<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width">
<meta name="theme-color" content="#222"><meta name="generator" content="Hexo 6.3.0">

  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">

<link rel="stylesheet" href="/css/main.css">



<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.2.1/css/all.min.css" integrity="sha256-Z1K5uhUaJXA7Ll0XrZ/0JhX4lAtZFpT6jkKrEDT0drU=" crossorigin="anonymous">
  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/animate.css/3.1.1/animate.min.css" integrity="sha256-PR7ttpcvz8qrF57fur/yAx1qXMFJeJFiA6pSzWi0OIE=" crossorigin="anonymous">

<script class="next-config" data-name="main" type="application/json">{"hostname":"example.com","root":"/","images":"/images","scheme":"Muse","darkmode":false,"version":"8.14.1","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12},"copycode":{"enable":false,"style":null},"bookmark":{"enable":false,"color":"#222","save":"auto"},"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"stickytabs":false,"motion":{"enable":true,"async":false,"transition":{"menu_item":"fadeInDown","post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"prism":false,"i18n":{"placeholder":"搜索...","empty":"没有找到任何搜索结果：${query}","hits_time":"找到 ${hits} 个搜索结果（用时 ${time} 毫秒）","hits":"找到 ${hits} 个搜索结果"},"path":"/search.xml","localsearch":{"enable":true,"trigger":"auto","top_n_per_article":-1,"unescape":false,"preload":false}}</script><script src="/js/config.js"></script>

    <meta property="og:type" content="website">
<meta property="og:title" content="JsyBlog">
<meta property="og:url" content="http://example.com/page/21/index.html">
<meta property="og:site_name" content="JsyBlog">
<meta property="og:locale" content="zh_CN">
<meta property="article:author" content="SongyangJi">
<meta name="twitter:card" content="summary">


<link rel="canonical" href="http://example.com/page/21/">



<script class="next-config" data-name="page" type="application/json">{"sidebar":"","isHome":true,"isPost":false,"lang":"zh-CN","comments":"","permalink":"","path":"page/21/index.html","title":""}</script>

<script class="next-config" data-name="calendar" type="application/json">""</script>
<title>JsyBlog</title>
  








  <noscript>
    <link rel="stylesheet" href="/css/noscript.css">
  </noscript>
</head>

<body itemscope itemtype="http://schema.org/WebPage" class="use-motion">
  <div class="headband"></div>

  <main class="main">
    <div class="column">
      <header class="header" itemscope itemtype="http://schema.org/WPHeader"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏" role="button">
        <span class="toggle-line"></span>
        <span class="toggle-line"></span>
        <span class="toggle-line"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <i class="logo-line"></i>
      <h1 class="site-title">JsyBlog</h1>
      <i class="logo-line"></i>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger" aria-label="搜索" role="button">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>



<nav class="site-nav">
  <ul class="main-menu menu"><li class="menu-item menu-item-home"><a href="/" rel="section"><i class="fa fa-home fa-fw"></i>首页</a></li><li class="menu-item menu-item-tags"><a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>标签</a></li><li class="menu-item menu-item-categories"><a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>分类</a></li><li class="menu-item menu-item-archives"><a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>归档</a></li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>搜索
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup"><div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocapitalize="off" maxlength="80"
           placeholder="搜索..." spellcheck="false"
           type="search" class="search-input">
  </div>
  <span class="popup-btn-close" role="button">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div class="search-result-container no-result">
  <div class="search-result-icon">
    <i class="fa fa-spinner fa-pulse fa-5x"></i>
  </div>
</div>

    </div>
  </div>

</header>
        
  
  <aside class="sidebar">

    <div class="sidebar-inner sidebar-overview-active">
      <ul class="sidebar-nav">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <div class="sidebar-panel-container">
        <!--noindex-->
        <div class="post-toc-wrap sidebar-panel">
        </div>
        <!--/noindex-->

        <div class="site-overview-wrap sidebar-panel">
          <div class="site-author animated" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">SongyangJi</p>
  <div class="site-description" itemprop="description"></div>
</div>
<div class="site-state-wrap animated">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
        <a href="/archives/">
          <span class="site-state-item-count">251</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
          <a href="/categories/">
        <span class="site-state-item-count">45</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
          <a href="/tags/">
        <span class="site-state-item-count">109</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>

        </div>
      </div>
    </div>

    
  </aside>


    </div>

    <div class="main-inner index posts-expand">

    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/30/Redis%E7%B3%BB%E5%88%97%E7%AC%94%E8%AE%B0%E4%B9%8B%20%E2%80%94%E2%80%94%20%E7%BC%93%E5%AD%98%E4%B8%8E%E6%95%B0%E6%8D%AE%E5%BA%93%E4%B8%80%E8%87%B4%E6%80%A7/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/30/Redis%E7%B3%BB%E5%88%97%E7%AC%94%E8%AE%B0%E4%B9%8B%20%E2%80%94%E2%80%94%20%E7%BC%93%E5%AD%98%E4%B8%8E%E6%95%B0%E6%8D%AE%E5%BA%93%E4%B8%80%E8%87%B4%E6%80%A7/" class="post-title-link" itemprop="url">Redis系列笔记之 —— 缓存与数据库一致性</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>

      <time title="创建时间：2021-10-30 00:00:00" itemprop="dateCreated datePublished" datetime="2021-10-30T00:00:00+08:00">2021-10-30</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar-check"></i>
      </span>
      <span class="post-meta-item-text">更新于</span>
      <time title="修改时间：2021-11-30 08:14:54" itemprop="dateModified" datetime="2021-11-30T08:14:54+08:00">2021-11-30</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/Redis/" itemprop="url" rel="index"><span itemprop="name">Redis</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h2 id="使用缓存的两种模式"><a href="#使用缓存的两种模式" class="headerlink" title="使用缓存的两种模式"></a>使用缓存的两种模式</h2><h3 id="Cache-Aside-Pattern（旁路缓存模式）"><a href="#Cache-Aside-Pattern（旁路缓存模式）" class="headerlink" title="Cache-Aside Pattern（旁路缓存模式）"></a>Cache-Aside Pattern（旁路缓存模式）</h3><ol>
<li>失效：程序先从缓存中读取数据，如果没有命中，则从数据库中读取，成功之后将数据放到缓存中；</li>
<li>命中：程序先从缓存中读取数据，如果命中，则直接返回；</li>
<li>更新：程序<strong>先更新数据库，再删除缓存</strong>。（这个操作顺序之后在细讲）。</li>
</ol>
<h3 id="Write-behind（异步缓存写入）"><a href="#Write-behind（异步缓存写入）" class="headerlink" title="Write-behind（异步缓存写入）"></a>Write-behind（异步缓存写入）</h3><p>这种使用模式的读缓存机制和上是一样的。但是更新缓存的策略为：</p>
<p><strong>更新时只更新缓存，不直接更新数据库</strong>，通过<strong>批量异步</strong> 的方式来更新数据库。</p>
<p>这个设计的好处就是让数据的I&#x2F;O操作飞快无比（因为直接操作内存嘛 ），因为异步，write backg还可以合并对同一个数据的多次操作，所以性能的提高是相当可观的。</p>
<p>但是，其带来的问题是，数据不是强一致性的，而且可能会丢失（我们知道Unix&#x2F;Linux非正常关机会导致数据丢失，就是因为这个事）。</p>
<p>另外，Write Back实现逻辑比较复杂，因为他需要track有哪数据是被更新了的，需要刷到持久层上。操作系统的write back会在仅当这个cache需要失效的时候，才会被真正持久起来，比如，内存不够了，或是进程退出了等情况，这又叫lazy write。</p>
<p>这种方式下，缓存和数据库的一致性不强，<strong>对一致性要求高的系统要谨慎使用</strong> 。但是它适合频繁写的场景，MySQL的<strong>InnoDB Buffer Pool机制</strong> 就使用到这种模式。</p>
<blockquote>
<p>不过本篇文章只对第一种模式进行拓展解析。</p>
</blockquote>
<h2 id="具体的更新缓存策略"><a href="#具体的更新缓存策略" class="headerlink" title="具体的更新缓存策略"></a>具体的更新缓存策略</h2><ol>
<li>操作缓存的时候，到底是<strong>删除缓存</strong>呢，还是<strong>更新缓存</strong>？</li>
<li>双写的情况下，<strong>先操作数据库</strong>还是<strong>先操作缓存</strong>？</li>
</ol>
<p>组合起来就有四种情况：</p>
<ol>
<li>先更新缓存，再更新数据库</li>
<li>先更新数据库，再更新缓存</li>
<li>先删除缓存，再更新数据库</li>
<li>先更新数据库，再删除缓存</li>
</ol>
<h3 id="先更新缓存，再更新数据库"><a href="#先更新缓存，再更新数据库" class="headerlink" title="先更新缓存，再更新数据库"></a>先更新缓存，再更新数据库</h3><p>这种方式可轻易排除，因为如果先更新缓存成功，但是数据库更新失败，则肯定会造成数据不一致。</p>
<p>绝对不要使用。</p>
<h3 id="先更新数据库，再更新缓存"><a href="#先更新数据库，再更新缓存" class="headerlink" title="先更新数据库，再更新缓存"></a>先更新数据库，再更新缓存</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">updateDB();</span><br><span class="line">updateRedis();</span><br></pre></td></tr></table></figure>

<p>这种缓存更新策略俗称双写。</p>
<p><strong>存在问题是：并发更新数据库场景下，会将脏数据刷到缓存。</strong></p>
<p>注意上面的代码并非是原子的，在并发环境下，很明显很有可能出现数据不一致的情况。</p>
<p>比如下面的竞态条件：</p>
<p><img src="/2021/10/30/Redis%E7%B3%BB%E5%88%97%E7%AC%94%E8%AE%B0%E4%B9%8B%20%E2%80%94%E2%80%94%20%E7%BC%93%E5%AD%98%E4%B8%8E%E6%95%B0%E6%8D%AE%E5%BA%93%E4%B8%80%E8%87%B4%E6%80%A7/%E5%85%88%E6%9B%B4%E6%96%B0%E6%95%B0%E6%8D%AE%E5%BA%93%EF%BC%8C%E5%86%8D%E6%9B%B4%E6%96%B0%E7%BC%93%E5%AD%98.png" alt="先更新数据库，再更新缓存"></p>
<h3 id="先删除缓存，再更新数据库"><a href="#先删除缓存，再更新数据库" class="headerlink" title="先删除缓存，再更新数据库"></a>先删除缓存，再更新数据库</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">deleteRedis();</span><br><span class="line">updateDB();</span><br></pre></td></tr></table></figure>

<p><strong>存在问题：更新数据库之前，若有查询请求，会将脏数据刷到缓存。</strong></p>
<p><img src="/2021/10/30/Redis%E7%B3%BB%E5%88%97%E7%AC%94%E8%AE%B0%E4%B9%8B%20%E2%80%94%E2%80%94%20%E7%BC%93%E5%AD%98%E4%B8%8E%E6%95%B0%E6%8D%AE%E5%BA%93%E4%B8%80%E8%87%B4%E6%80%A7/%E5%85%88%E5%88%A0%E9%99%A4%E7%BC%93%E5%AD%98%EF%BC%8C%E5%86%8D%E6%9B%B4%E6%96%B0%E6%95%B0%E6%8D%AE%E5%BA%93.png" alt="先删除缓存，再更新数据库"></p>
<h3 id="先更新数据库，再删除缓存"><a href="#先更新数据库，再删除缓存" class="headerlink" title="先更新数据库，再删除缓存"></a>先更新数据库，再删除缓存</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">updateDB();</span><br><span class="line">deleteRedis();</span><br></pre></td></tr></table></figure>



<p><strong>存在问题：在更新数据库之前有查询请求，并且缓存失效了，会查询数据库，然后更新缓存。如果在查询数据库和更新缓存之间进行了数据库更新的操作，那么就会把脏数据刷到缓存。</strong></p>
<p><img src="/2021/10/30/Redis%E7%B3%BB%E5%88%97%E7%AC%94%E8%AE%B0%E4%B9%8B%20%E2%80%94%E2%80%94%20%E7%BC%93%E5%AD%98%E4%B8%8E%E6%95%B0%E6%8D%AE%E5%BA%93%E4%B8%80%E8%87%B4%E6%80%A7/%E5%85%88%E6%9B%B4%E6%96%B0%E6%95%B0%E6%8D%AE%E5%BA%93%EF%BC%8C%E5%86%8D%E5%88%A0%E9%99%A4%E7%BC%93%E5%AD%98.png" alt="先更新数据库，再删除缓存"></p>
<p>如果发生上述情况，确实是会发生脏数据。但是发生上述情况有一个先天性条件，就是写数据库操作比读数据库操作耗时更短。</p>
<p><strong>不过数据库的读操作的速度远快于写操作的</strong></p>
<p>因此这一情形导致数据不一致性的概率比上面3种可能性都要小。</p>
<h2 id="对上面四种方式的总结"><a href="#对上面四种方式的总结" class="headerlink" title="对上面四种方式的总结"></a>对上面四种方式的总结</h2><p>为什么无论按照哪种做法都有可能导致数据不一致？</p>
<p>上面的四种策略其实到最后<strong>有两步操作</strong>，而<strong>这两步操作并不是原子性的</strong>，在并发场景下总会导致缓存和数据库的不一致。 </p>
<h3 id="更新缓存还是删除缓存？"><a href="#更新缓存还是删除缓存？" class="headerlink" title="更新缓存还是删除缓存？"></a>更新缓存还是删除缓存？</h3><p>对于一个不能保证事务性的操作，一定涉及“哪个任务先做，哪个任务后做”的问题，解决这个问题的方向是：如果出现不一致，谁先做对业务的影响较小，就谁先执行。</p>
<p>更新缓存：</p>
<p>缺点：</p>
<ul>
<li><strong>更新缓存需要有一定的维护成本</strong>；</li>
<li>存在<strong>并发更新</strong>的问题；</li>
<li>写多读少的情况下，读请求还没有来，<strong>缓存会被无用地更新很多次</strong>，没有起到缓存的作用；</li>
</ul>
<p>删除缓存</p>
<ul>
<li>优点：简单、成本低，容易开发；</li>
<li>缺点：会造成一次cache miss；</li>
</ul>
<p>如果更新缓存开销较小并且读多写少，基本不会有高写并发的时候可以才用更新缓存，否则<strong>通用做法还是删除缓存</strong>。</p>
<h2 id="推荐的做法"><a href="#推荐的做法" class="headerlink" title="推荐的做法"></a>推荐的做法</h2><h3 id="设置一个合理的redis过期时间"><a href="#设置一个合理的redis过期时间" class="headerlink" title="设置一个合理的redis过期时间"></a>设置一个合理的redis过期时间</h3><p><strong>缓存必须要有过期时间</strong>。</p>
<p>为什么必须要有过期时间？</p>
<p>首先，对于缓存来说，当它的命中率越高的时候，我们的系统性能也就越好。如果某个缓存项没有过期时间，而它命中的概率又很低，这就是在浪费缓存的空间。</p>
<p>其次，设置过期时间还有一个最大好处，就是当数据库跟缓存出现数据不一致的情况时，这个可以<strong>作为一个最后的兜底手段</strong>。也就是说，当数据确实出现不一致的情况时，过期时间可以保证只有在出现不一致的时间点到缓存过期这段时间之内，数据库跟缓存的数据是不一致的，因此也<strong>保证了数据的最终一致性</strong>。</p>
<p>所以，如果可以设置一个比较小的过期时间，那么<strong>数据不一致终归是暂时的，最终一致性可以保证</strong>。</p>
<h3 id="延迟双删"><a href="#延迟双删" class="headerlink" title="延迟双删"></a>延迟双删</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">write</span><span class="params">(String key,Object data)</span>&#123;</span><br><span class="line">     redis.del(key);  <span class="comment">// 第一次删除缓存</span></span><br><span class="line">     db.update(data); <span class="comment">// 操作数据库</span></span><br><span class="line">     Thread.sleep(<span class="number">1000</span>);  <span class="comment">// 延时一段时间，在这段时间内有可能读操作设置的redis脏数据</span></span><br><span class="line">     redis.del(key); <span class="comment">// 第二次删除缓存</span></span><br><span class="line"> &#125;</span><br></pre></td></tr></table></figure>



<p>大家应该评估自己的项目的读数据业务逻辑的耗时，然后写数据的休眠时间，则在读数据业务逻辑的耗时基础上即可加几百 ms 即可。</p>
<p>这么做的目的，就是<strong>尽可能确保读请求结束，写请求可以删除读请求造成的缓存脏数据。</strong>（也就是说为了避免产生第四种策略的问题）</p>
<p>当然，完全可能在第二次删除缓存之后，还有读操作设置了旧的缓存数据。</p>
<p>是的，所以<strong>延迟双删这种方案说到底也只是降低了数据不一致的概率，不是强一致性</strong>。</p>
<h3 id="延时的具体实现"><a href="#延时的具体实现" class="headerlink" title="延时的具体实现"></a>延时的具体实现</h3><p>上面的阻塞只是语义上的延时，实际不会这样做，否则这个用户线程就阻塞在这了。</p>
<p>实际处理时，可以如下方法：</p>
<ol>
<li>线程池提交任务，任务中睡眠</li>
<li>Quartz定时任务</li>
<li>Rabbitmq的延时队列</li>
<li>Java中的DelayedQueue</li>
</ol>
<h3 id="删除缓存失败的重试机制"><a href="#删除缓存失败的重试机制" class="headerlink" title="删除缓存失败的重试机制"></a>删除缓存失败的重试机制</h3><p>采取<strong>先更新数据库，再删除缓存</strong>这种方案，我们没有考虑操作数据库或者操作缓存可能失败的情况，而这种情况也是客观存在的。那么在这里我们简单讨论下，首先是如果更新数据库失败了，其实没有太大关系，因为此时数据库和缓存中都还是老数据，不存在不一致的问题。</p>
<p>假设删除缓存失败了呢？此时确实会存在数据不一致的情况。除了设置缓存过期时间这种兜底方案之外，如果我们希望尽可能保证缓存可以被及时删除，那么我们必须<strong>要考虑对删除操作进行重试</strong>。</p>
<p>最简单方式，就是在代码中进行重试删除，但是这样的代码侵入比较严重。</p>
<p>另外考虑到，分布式环境下一台需要删除redis缓存的机器可能会宕机，导致需要删除的缓存没有得到正确删除，此时需要删除的key的信息就会丢失。</p>
<p>这个时候可以引入 <strong>消息中间件</strong>, 将需要删除的key发往消息中间件，然后消费端负责根据拉下来的key删除缓存，删除失败就把这条消息重新入队，直到成功。</p>
<p><img src="/2021/10/30/Redis%E7%B3%BB%E5%88%97%E7%AC%94%E8%AE%B0%E4%B9%8B%20%E2%80%94%E2%80%94%20%E7%BC%93%E5%AD%98%E4%B8%8E%E6%95%B0%E6%8D%AE%E5%BA%93%E4%B8%80%E8%87%B4%E6%80%A7/del-key.png" alt="图片"></p>
<p>上面的做法的确点就是对业务代码有入侵，可以采用一种订阅数据库binlog的方式，用非业务代码完成对缓存的删除。</p>
<p><img src="/2021/10/30/Redis%E7%B3%BB%E5%88%97%E7%AC%94%E8%AE%B0%E4%B9%8B%20%E2%80%94%E2%80%94%20%E7%BC%93%E5%AD%98%E4%B8%8E%E6%95%B0%E6%8D%AE%E5%BA%93%E4%B8%80%E8%87%B4%E6%80%A7/binlog-mq.png" alt="img"></p>
<p>启动一个订阅程序去订阅数据库的 binlog，获得需要操作的数据。在应用程序中，另起一段程序，获得这个订阅程序传来的信息，进行删除缓存操作。</p>
<h2 id="怎么做到强一致性？"><a href="#怎么做到强一致性？" class="headerlink" title="怎么做到强一致性？"></a>怎么做到强一致性？</h2><p>上面讲述的所有的方案都是最终一致性的， 如果我们就要实现强一致性呢？</p>
<h3 id="读写锁"><a href="#读写锁" class="headerlink" title="读写锁"></a>读写锁</h3><p>假设我们仍然使用<code> Cache-Aside Pattern</code>, 所谓导致数据不一致的情况</p>
<p>实际上根本原因是<strong>写操作的两个方法</strong>（更新数据库-删除缓存）与<strong>读操作两个方法</strong>（查询数据库-更新缓存）不是串行化的执行。</p>
<p>所以，为了做到这样的强一致性，我们可以引入<strong>读写锁</strong>，去完成这样<strong>写读-读写-读读-写写</strong>操作之间的串行化操作。</p>
<p>使用读写锁之后：</p>
<p>回到”先更新数据库，再删除缓存”这个方案本身上来，从字面上来看，这里有两步操作，因此在数据库更新之前，到缓存被删除这段时间之内，读请求读取到的都是脏数据。</p>
<p>如果要实现这两者的强一致性，只能是在更新完数据库之前，所有的读请求都必须要被阻塞直到缓存最终被删除为止，读请求和读请求之间并不没有冲突，而这个同步语义正由<strong>读写锁</strong>提供。</p>
<p>需要注意是，</p>
<ol>
<li>要使用分布式读写锁；</li>
<li>锁的粒度要尽可能小。</li>
</ol>
<h3 id="分布式一致性算法"><a href="#分布式一致性算法" class="headerlink" title="分布式一致性算法"></a>分布式一致性算法</h3><p>虽然这里谈论的是Redis缓存和数据库的数据一致性，但是逻辑模型上这里和CAP定理中的C是一回事。</p>
<p>因而在分布式系统中为了达到多副本单操作的一致性的各种一致性算法都可以派上用场，来达到这里的缓存与数据库一致性的目的。</p>
<p>比如2PC、3PC、Paxos和Raft 等。</p>
<p>说到这里，你可以看到为了做到 <strong>缓存-数据库一致性</strong>的强一致性并不是一件很容易的事，实际上分布式环境下的一致性问题一直都是一个痛点，如果你使用了redis缓存来加快数据库多读少写场景下的业务，同时又想确保<strong>缓存-数据库一致性</strong>，是不现实的，或者说实现成本太高，即使实现效果也不好。</p>
<p>这个时候，你需要考虑，如果真的就需要保证强一致性，还不如将读写串行化，甚至去掉缓存。</p>
<h2 id="分布式架构、读写分离环境下的一致性保证"><a href="#分布式架构、读写分离环境下的一致性保证" class="headerlink" title="分布式架构、读写分离环境下的一致性保证"></a>分布式架构、读写分离环境下的一致性保证</h2><p>上面的谈论的缓存-数据库一致性的架构仍然是单数据库-单应用程序的架构，如果是数据库读写分离环境下的呢？</p>
<p>我们知道数据库（以Mysql为例）主从之间的数据同步是通过binlog同步来实现的，因此这里可以考虑订阅binlog（可以使用canal之类的中间件实现，提取出要删除的缓存项，然后作为消息写入消息队列，然后再由消费端进行慢慢的消费去删除redis缓存和重试。</p>
<p><img src="/2021/10/30/Redis%E7%B3%BB%E5%88%97%E7%AC%94%E8%AE%B0%E4%B9%8B%20%E2%80%94%E2%80%94%20%E7%BC%93%E5%AD%98%E4%B8%8E%E6%95%B0%E6%8D%AE%E5%BA%93%E4%B8%80%E8%87%B4%E6%80%A7/redis-mysql.png"></p>
<blockquote>
<p><a target="_blank" rel="noopener" href="https://juejin.cn/post/6844903941646319623#heading-10">聊聊数据库与缓存数据一致性问题</a></p>
<p>[缓存更新的套路](Redis系列笔记之 —— 缓存与数据库一致性&#x2F;17416.html)</p>
<p><a target="_blank" rel="noopener" href="https://xie.infoq.cn/article/47241d099404a1565e168fad4">缓存与数据库一致性问题深度剖析</a></p>
</blockquote>

      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/29/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94InnoDB%20Buffer%20Pool/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/29/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94InnoDB%20Buffer%20Pool/" class="post-title-link" itemprop="url">MySQL系列——InnoDB Buffer Pool</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>

      <time title="创建时间：2021-10-29 10:03:19" itemprop="dateCreated datePublished" datetime="2021-10-29T10:03:19+08:00">2021-10-29</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar-check"></i>
      </span>
      <span class="post-meta-item-text">更新于</span>
      <time title="修改时间：2021-11-03 16:12:02" itemprop="dateModified" datetime="2021-11-03T16:12:02+08:00">2021-11-03</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/MySQL/" itemprop="url" rel="index"><span itemprop="name">MySQL</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h1 id="InnoDB-Buffer-Pool"><a href="#InnoDB-Buffer-Pool" class="headerlink" title="InnoDB Buffer Pool"></a>InnoDB Buffer Pool</h1><h2 id="介绍"><a href="#介绍" class="headerlink" title="介绍"></a>介绍</h2><p>磁盘太慢，用内存作为缓冲区很有必要——这是缓存的基本思想，将数据存放在读写速度更快的存储介质中。</p>
<p>不过这里的缓冲区指的不是 Redis、Memchached这些外部的内存性缓冲，还是MySQL自治的一块内存缓冲区。</p>
<p>所谓自治，就是说完全由MySQL自己去管理这块内存区域，自己完成脏数据的刷盘，使用MySQL执行CRUD无需也无法自己做这方面的操作——不过你可以修改其中的配置以调整缓存的具体行为。</p>
<p>InnoDB Buffer Pool 本质上是MySQL在向操作系统申请的一大块内存，默认是128MB（比较小，完全可以开大一点）。</p>
<h2 id="内部组成"><a href="#内部组成" class="headerlink" title="内部组成"></a>内部组成</h2><p>Buffer Pool 对应的一大块内存被划分一个又一个页面，页面大小和InnoDB表空间的页面大小一致，默认是16KB，不妨称它为<strong>缓冲页。</strong></p>
<p>由于这些缓冲业本身较大，如果已过用户的调整可能更大，不便于直接管理。于是为每个缓冲页创建一个控制块，</p>
<p>每个控制块包含该页所属的表空间编号、页号、缓冲页在Buffer Pool 中的地址等关键信息。</p>
<p>每个控制块的所占用的内存大小是一致的，大概只有16KB的5%左右（aka 800个字节）。</p>
<img src="innodb-buffer.png" style="zoom:35%;" />





<h2 id="free-链表"><a href="#free-链表" class="headerlink" title="free 链表"></a>free 链表</h2><p>缓冲页一开始都为空（逻辑上的空），随着从磁盘读取数据会逐渐使用buffer pool中的页。</p>
<p>如何区分这些空闲的页和非空闲的页呢？</p>
<p>MySQL的做法并不特殊，使用一条链表将空闲也组织起来。</p>
<p>具体的数据结构，下面的图说的很清楚了。</p>
<p>红色对应的空闲的控制块，蓝色对应的是相应的缓冲页。</p>
<img src="free.png" style="zoom:40%;" />



<h2 id="flush-链表"><a href="#flush-链表" class="headerlink" title="flush 链表"></a>flush 链表</h2><p>读缓存的逻辑比较简单，就是把磁盘中的页读到内存中（不过还要设计缓冲页的淘汰问题）。</p>
<p>写缓存呢？</p>
<p>和上面，我们需要把buffer pool中的写过的页（也就是和磁盘的上的页不一致的页），也就是脏页管理起来，然后按照某种刷盘逻辑将这些脏页在未来某个时刻写到磁盘上。</p>
<p>具体的组织结构也是链表，和上面的的形式几乎一致。不再赘述。</p>
<h2 id="缓冲页的哈希表"><a href="#缓冲页的哈希表" class="headerlink" title="缓冲页的哈希表"></a>缓冲页的哈希表</h2><p>如何根据快速知道某个是否在缓冲池中，毫无疑问，哈希表可以派上用场。</p>
<p>具体的，可以使用 <strong>表空间号+页号</strong> 作为键来定位一个页，值是页控制块的内存地址。</p>
<h2 id="LRU链表的使用管理"><a href="#LRU链表的使用管理" class="headerlink" title="LRU链表的使用管理"></a>LRU链表的使用管理</h2><p>只要涉及到缓存，就必然涉及到缓存区不够用，需要淘汰部分缓存的情况。对于innodb buffer pool来说也不意外。</p>
<p>不过，这里的LRU算法不是最朴素的实现（比如每使用一次页，就把它对应的控制块移动到LRU链表的头部），<br>这里的LRU链表是变种。</p>
<h3 id="数据冷热分离"><a href="#数据冷热分离" class="headerlink" title="数据冷热分离"></a>数据冷热分离</h3><p>为什么呢？</p>
<p>两个原因：</p>
<ol>
<li>MySQL的预读策略可能将一些原本不会使用的页面误装载进了buffer pool；</li>
<li>执行一些全表扫描的SQL时，会将大量的数据页加载进buffer pool，不过又由于这些全表扫描出的页面的本身的使用频率的非常低（全表的扫描这样的sql一般只有定期做报表的时候用到），这个时候相当于一下子就把缓冲区全用冷数据充满了，也就是说相当于清空了缓冲区。</li>
</ol>
<p>所以，MySQL对这里LRU链表作如下处理，</p>
<p>将数据冷热分离，默认情况下，将LRU链表前约5&#x2F;8的数据作为热数据（也称为 young ）区域，后3&#x2F;8的数据作为冷数据（也称为 old）区域。</p>
<p>如何对上面提出的两个问题做出优化，上面的问题的共同点在于使用一个页面之后就再不用了（aka，用完即弃，不是那种在一个时间段里频繁使用到的数据）。</p>
<p>所以，<code>innnodb_old_blocks_time</code>这个系统配置变量其到作用，默认是1000ms。</p>
<p>对于从磁盘加载进来的数据页，先将它加入old区域，记录下第一次加入的时间，之后再访问它的时候，如果间隔的时间小于<code>innnodb_old_blocks_time</code>, 还不会加入young区域，仍然作为冷数据处理。</p>
<p>于是，对于那些原表扫描和预读机制的冷数据页不会影响到真正的需要缓存的热数据。</p>
<h3 id="进一步优化"><a href="#进一步优化" class="headerlink" title="进一步优化"></a>进一步优化</h3><p>是不是对于young区域数据页每访问一次就要把它加入LRU链表的头部呢？因为这些数据可能都很热，每一次每访问一次就把要移动一次控制块节点，没必要。</p>
<p>所以，MySQL只对于那些处于young区域后3&#x2F;4区域的节点才会在使用的时候加入young的头部。</p>
<h2 id="刷新脏页到磁盘"><a href="#刷新脏页到磁盘" class="headerlink" title="刷新脏页到磁盘"></a>刷新脏页到磁盘</h2><p>后台线程会定期将脏页刷新到磁盘中。</p>
<p>也就是说，这里的缓存使用方式实际上是<strong>Write-hehind（异步缓存写入）</strong>，适合多写的方式。</p>
<p>脏页的刷新主要有3种做法：</p>
<ol>
<li><p>从LRU的尾部（也是从old区域）开始扫描一些页面，扫描页面的数量由<code>innodb_lru_scan_depth</code>（默认是1024）指定，如果扫描中发现是脏页就把它刷新到磁盘。</p>
</li>
<li><p>从flush链表刷新一部分页面到磁盘。刷新的速率取决于当时系统繁忙的程度。</p>
</li>
<li><p>如果用户线程想将一个磁盘页加载进buffer pool，但是此时全都是脏页了，这时就必须挑出一个脏页刷到磁盘上。</p>
</li>
</ol>
<h2 id="查看当前-buffer-pool的状态-x2F-配置"><a href="#查看当前-buffer-pool的状态-x2F-配置" class="headerlink" title="查看当前 buffer pool的状态&#x2F;配置"></a>查看当前 buffer pool的状态&#x2F;配置</h2><p>查看当前状态</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">show</span> engine innodb status;</span><br></pre></td></tr></table></figure>

<p>输出</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">......</span></span><br><span class="line">BUFFER POOL AND MEMORY</span><br><span class="line">----------------------</span><br><span class="line">Total large memory allocated 137428992 # 向操作系统申请的一大块连续空间 + 全部控制块 + 缓冲页 + 碎片</span><br><span class="line">Dictionary memory allocated 410319</span><br><span class="line">Buffer pool size   8191  # 一共容纳多少缓冲页（页数）</span><br><span class="line">Free buffers       7017  # 空闲的缓冲页个数</span><br><span class="line">Database pages     1170  # LRU链表中的页的数量</span><br><span class="line">Old database pages 451   # OLD区域中的页的数量</span><br><span class="line">Modified db pages  0     # FLUSH链表中页的数量（脏页的数量）</span><br><span class="line">Pending reads      0</span><br><span class="line">Pending writes: LRU 0, flush list 0, single page 0</span><br><span class="line">Pages made young 0, not young 0</span><br><span class="line">0.00 youngs/s, 0.00 non-youngs/s</span><br><span class="line">Pages read 1039, created 131, written 137</span><br><span class="line">0.00 reads/s, 0.00 creates/s, 0.00 writes/s</span><br><span class="line">No buffer pool page gets since the last printout</span><br><span class="line">Pages read ahead 0.00/s, evicted without access 0.00/s, Random read ahead 0.00/s</span><br><span class="line">LRU len: 1170, unzip_LRU len: 0</span><br><span class="line">I/O sum[0]:cur[0], unzip sum[0]:cur[0]</span><br></pre></td></tr></table></figure>

<p>具体的参看官方文档。</p>
<p><strong>查看配置变量</strong></p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">show</span> status <span class="keyword">like</span> <span class="string">&#x27;Innodb_buffer_pool_%&#x27;</span></span><br></pre></td></tr></table></figure>



<p><strong>变量解析</strong></p>
<ul>
<li>Innodb_buffer_pool_pages_total 参数表示缓存页面的总数量;</li>
<li>Innodb_buffer_pool_pages_data 代表有数据的缓存页数;</li>
<li>Innodb_buffer_pool_pages_free 代表没有使用的缓存页数;</li>
<li>Innodb_buffer_pool_pages_misc: innodb buffer pool 缓存池中当前已经被用作管理用途或hash index而不能用作为普通数据页的数目。</li>
<li>Innodb_buffer_pool_read_requests表示read请求的次数，</li>
<li>Innodb_buffer_pool_reads 表示从物理磁盘中读取数据的请求次数，</li>
</ul>
<p>innodb buffer的read命中率 &#x3D;（Innodb_buffer_pool_read_requests -Innodb_buffer_pool_reads） &#x2F; Innodb_buffer_pool_read_requests * 100%。<br>如果这个命中率小于95%，建议增大 innodb_buffer_pool_size。</p>
<p>如果Innodb_buffer_pool_pages_free偏大的话，证明有很多缓存没有被利用到，这时可以考虑减小缓存; 相反Innodb_buffer_pool_pages_data过大就考虑增大缓存。</p>
<h2 id="配置-Buffer-Pool"><a href="#配置-Buffer-Pool" class="headerlink" title="配置 Buffer Pool"></a>配置 Buffer Pool</h2><h3 id="配置解析"><a href="#配置解析" class="headerlink" title="配置解析"></a>配置解析</h3><p>由于一个Buffer Pool可能很大，一次性申请、移动、管理它不适合，所以MySQL实际做法是用多个做个chunk组成一个buffer pool。</p>
<ul>
<li>innodb_buffer_pool_size 参数为innodb_buffer_pool的大小设置（默认是128MB）。</li>
<li>innodb_buffer_pool_chunk_size 参数为InnoDB缓冲池块大小（默认是128MB）。</li>
<li>innodb_buffer_pool_instances 参数为缓冲池实例的个数（默认是1）。</li>
</ul>
<p>注意，不同的innodb_buffer_pool 实例的链表管理、哈希管理、锁管理都是独立的，也正因此如果可以配置多个buffer pool实例可以增大并发度。</p>
<p>配置规则：<br>*<em>innodb_buffer_pool_size &#x3D; innodb_buffer_pool_chunk_size * innodb_buffer_pool_instances <em>N</em></em></p>
<p>其中N指的是一个buffer pool实例中有几个 chunk。</p>
<p>（注意，如果不是整数倍，那么会调整为整数倍；又如果配置的innodb_buffer_pool_size 小于innodb_buffer_pool_chunk_size * innodb_buffer_pool_instances，那么MySQL会缩小innodb_buffer_pool_chunk_size的大小）</p>
<p>系统默认的innodb_buffer_pool_chunk_size （8.x版本）为128M。<br>innodb_buffer_pool_instances参数的默认设置为1 最大设置为64 ，但是<br>将innodb_buffer_pool_size大小设置为1GB或更大时，此选项才生效。<br>（主要是防止有太多小的instance从而导致性能问题。）</p>
<h3 id="配置参考"><a href="#配置参考" class="headerlink" title="配置参考"></a>配置参考</h3><p>建议设置为系统内存的50%-80%，但也不是越大越好，要根据具体项目具体分析<br>（操作系统留1G左右，mysql连接数*4M，宿主程序缓存nM）。</p>
<table>
<thead>
<tr>
<th align="left">实例内存大小（单位：MB）</th>
<th align="left">默认Buffer Pool（单位：MB）</th>
<th align="left">推荐最大Buffer Pool（单位：MB）</th>
</tr>
</thead>
<tbody><tr>
<td align="left">1024</td>
<td align="left">256</td>
<td align="left">256</td>
</tr>
<tr>
<td align="left">2048</td>
<td align="left">512</td>
<td align="left">512</td>
</tr>
<tr>
<td align="left">4096</td>
<td align="left">1536</td>
<td align="left">1536</td>
</tr>
<tr>
<td align="left">8192</td>
<td align="left">4608</td>
<td align="left">4608</td>
</tr>
<tr>
<td align="left">16384</td>
<td align="left">12288</td>
<td align="left">12288</td>
</tr>
<tr>
<td align="left">24576</td>
<td align="left">18432</td>
<td align="left">19456</td>
</tr>
<tr>
<td align="left">32768</td>
<td align="left">24576</td>
<td align="left">25600</td>
</tr>
<tr>
<td align="left">49152</td>
<td align="left">36864</td>
<td align="left">38912</td>
</tr>
<tr>
<td align="left">65536</td>
<td align="left">49152</td>
<td align="left">52224</td>
</tr>
<tr>
<td align="left">98304</td>
<td align="left">73728</td>
<td align="left">77824</td>
</tr>
<tr>
<td align="left">131072</td>
<td align="left">98304</td>
<td align="left">104448</td>
</tr>
<tr>
<td align="left">196608</td>
<td align="left">147456</td>
<td align="left">156672</td>
</tr>
<tr>
<td align="left">229376</td>
<td align="left">172032</td>
<td align="left">183296</td>
</tr>
<tr>
<td align="left">262144</td>
<td align="left">196608</td>
<td align="left">208896</td>
</tr>
</tbody></table>
<h1 id="参考链接"><a href="#参考链接" class="headerlink" title="参考链接"></a>参考链接</h1><blockquote>
<p>《MySQL是怎样运行的》</p>
</blockquote>
<p><a target="_blank" rel="noopener" href="https://www.cnblogs.com/frankcui/p/15227775.html">https://www.cnblogs.com/frankcui/p/15227775.html</a></p>

      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E5%AD%98%E5%82%A8%E5%BC%95%E6%93%8E/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E5%AD%98%E5%82%A8%E5%BC%95%E6%93%8E/" class="post-title-link" itemprop="url">MySQL系列——存储引擎</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>
      

      <time title="创建时间：2021-10-27 15:21:44 / 修改时间：16:16:59" itemprop="dateCreated datePublished" datetime="2021-10-27T15:21:44+08:00">2021-10-27</time>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          
      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E7%B4%A2%E5%BC%95/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E7%B4%A2%E5%BC%95/" class="post-title-link" itemprop="url">MySQL系列——索引</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>
      

      <time title="创建时间：2021-10-27 14:08:19 / 修改时间：16:30:02" itemprop="dateCreated datePublished" datetime="2021-10-27T14:08:19+08:00">2021-10-27</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/MySQL/" itemprop="url" rel="index"><span itemprop="name">MySQL</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h1 id="索引的数据结构"><a href="#索引的数据结构" class="headerlink" title="索引的数据结构"></a>索引的数据结构</h1><h2 id="B树、B-树"><a href="#B树、B-树" class="headerlink" title="B树、B+树"></a>B树、B+树</h2><blockquote>
<p>关于 B树、B+树本身的介绍，不是本文的重点，之后也许会专门写一篇文章详细介绍，</p>
</blockquote>
<p>这里直接就贴出wiki的链接了。</p>
<p><a target="_blank" rel="noopener" href="https://zh.wikipedia.org/wiki/B%E6%A0%91">B树</a></p>
<p><a target="_blank" rel="noopener" href="https://zh.wikipedia.org/wiki/B%2B%E6%A0%91">B+树</a></p>
<ul>
<li><p>为什么不使用普通的平衡二叉树（如RB树、AVL树）？</p>
<p>MySQL的数据是存储在磁盘文件中的，查询处理数据时，需要先把磁盘中的数据加载到内存中，<strong>磁盘IO 操作非常耗时</strong>，所以我们优化的重点就是尽量减少磁盘 IO 操作。<strong>访问二叉树的每个节点就会发生一次IO，如果想要减少磁盘IO操作，就需要尽量降低树的高度</strong>。那如何降低树的高度呢？</p>
<p>B树的特点就是一个多叉的平衡树，特点就是<strong>矮胖</strong>（相比于普通平衡二叉树的高瘦），在相同数据规模下访问节点的次数更少，换句话说，磁盘IO的开销更少。</p>
</li>
</ul>
<p>如下图，红色的是主键，黄色的其余的数据，蓝色的是指向子节点的指针。</p>
<p><img src="/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E7%B4%A2%E5%BC%95/btree.png" alt="B树"></p>
<ul>
<li><p>为什么Innodb引擎使用B+树而不是B树？</p>
<ol>
<li><p>B树不支持范围查询的快速查找，你想想这么一个情况如果我们想要查找10和35之间的数据，查找到15之后，需要回到根节点重新遍历查找，需要从根节点进行多次遍历，查询效率有待提高。</p>
</li>
<li><p>如果data存储的是行记录，行的大小随着列数的增多，所占空间会变大。这时，<strong>一个页中可存储的数据量就会变少，树相应就会变高</strong>，磁盘IO次数就会变大。</p>
</li>
</ol>
<p>B+树和B树最主要的区别在于<strong>非叶子节点是否存储数据</strong>的问题</p>
<ul>
<li>B树：非叶子节点和叶子节点都会存储数据。</li>
<li>B+树：<strong>只有叶子节点才会存储数据，非叶子节点至存储键值</strong>。叶子节点之间使用双向指针连接，<strong>最底层的叶子节点形成了一个双向有序链表</strong>。</li>
</ul>
<p>当然，非要讲B+树和B树的缺点，就是每一次查询行数据，都一定要查询到叶子节点，不过这个缺点和它的优势相比可以忽略了。</p>
</li>
</ul>
<p><img src="/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E7%B4%A2%E5%BC%95/b+tree.png" alt="B+树"></p>
<h2 id="MyISAM引擎中的索引"><a href="#MyISAM引擎中的索引" class="headerlink" title="MyISAM引擎中的索引"></a>MyISAM引擎中的索引</h2><p>MyISAM的数据文件和索引文件是分开存储的。MyISAM使用B+树构建索引树时，<strong>叶子节点中存储的键值为索引列的值，数据为索引所在行的磁盘地址</strong>。</p>
<p><strong>在 MyISAM 中,辅助索引和主键索引的结构是一样的，没有任何区别</strong>。叶子节点的数据存储的都是行记录的磁盘地址。只是主键索引的键值是唯一的，而辅助索引的键值可以重复。</p>
<p><img src="/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E7%B4%A2%E5%BC%95/myisam.png" alt="MyISAM主键索引"></p>
<p>MyISAM通过key_buffer<strong>把索引先缓存到内存中</strong>，当需要访问数据时（通过索引访问数据），在内存中直接搜索索引，然后<strong>通过索引找到磁盘相应数据</strong>。</p>
<p>这也就是为什么索引不在key buffer命中时，速度慢的原因。</p>
<h2 id="InnoDB引擎中的索引"><a href="#InnoDB引擎中的索引" class="headerlink" title="InnoDB引擎中的索引"></a>InnoDB引擎中的索引</h2><blockquote>
<p>首先需要澄清的一点是，MySQL 跟 B+ 树没有直接的关系，<strong>真正与 B+ 树有关系的是 MySQL 的默认存储引擎 InnoDB</strong>，MySQL 中存储引擎的主要作用是负责数据的存储和提取，除了 InnoDB 之外，MySQL 中也支持 MyISAM 作为表的底层存储引擎。</p>
<p><del>不过本文不介绍xianMyISAM，只介绍 InnoDB，会在介绍MySQL中的存储引擎文章中介绍 MyISAM引擎以及其索引特点。</del></p>
</blockquote>
<h3 id="主键索引（聚簇索引）"><a href="#主键索引（聚簇索引）" class="headerlink" title="主键索引（聚簇索引）"></a>主键索引（聚簇索引）</h3><p>通俗点讲：</p>
<p>聚簇索引的意思就是：将数据存储与索引放到了一块，找到索引也就找到了数据。</p>
<p>每个InnoDB表都有一个聚簇索引 ，<strong>聚簇索引使用B+树构建</strong>，<strong>叶子节点存储的数据是整行记录</strong>。一般情况下，<strong>聚簇索引基本上等同于主键索引</strong>，当一个表没有创建主键索引时，InnoDB会自动创建一个ROWID字段来构建聚簇索引。</p>
<p>InnoDB创建索引的具体规则如下：</p>
<ol>
<li>在表上定义主键PRIMARY KEY，InnoDB将<strong>主键</strong>用作聚簇索引。</li>
<li>如果表没有定义主键，InnoDB会选择<strong>第一个不为NULL的唯一索引列</strong>用作聚簇索引。</li>
<li>如果以上两个都没有，InnoDB 会使用一个6 字节长整型的隐式字段 <strong>ROWID字段</strong>构建聚簇索引。该ROWID字段会在插入新行时自动递增。</li>
</ol>
<p> 在检索时，InnoDB使用此主键值在聚簇索引中搜索行记录。</p>
<p><img src="/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E7%B4%A2%E5%BC%95/innodb1.png" alt="InnoDB主键索引"></p>
<h3 id="二级索引（辅助索引）"><a href="#二级索引（辅助索引）" class="headerlink" title="二级索引（辅助索引）"></a>二级索引（辅助索引）</h3><p>除聚簇索引之外的所有索引都称为辅助索引。在中InnoDB，<strong>辅助索引中的叶子节点存储的数据是该行的主键值</strong>。</p>
<p><strong>辅助索引也使用B+树构建</strong>。</p>
<p><img src="/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E7%B4%A2%E5%BC%95/innodb2.png" alt="InnoDB辅助索引"></p>
<p>如何使用二级索引来查询行数据呢？</p>
<p>第一步，在二级索引里根据索引列查询到主键值。</p>
<p>第二步，根据主键值再到聚簇索引里查询数据行。</p>
<p>这个操作就叫做<strong>回表</strong>。</p>
<p><img src="/2021/10/27/MySQL%E7%B3%BB%E5%88%97%E2%80%94%E2%80%94%E7%B4%A2%E5%BC%95/rolltable.png" alt="回表"></p>
<h2 id="聚簇索引-vs-非聚簇索引"><a href="#聚簇索引-vs-非聚簇索引" class="headerlink" title="聚簇索引 vs 非聚簇索引"></a>聚簇索引 vs 非聚簇索引</h2><h3 id="优势"><a href="#优势" class="headerlink" title="优势"></a>优势</h3><ul>
<li>由于行数据和叶子节点存储在一起，同一页中会有多条行数据，访问同一数据页不同行记录时，已经把页加载到了Buffer中，再次访问的时候，会在内存中完成访问，不必访问磁盘。</li>
<li>辅助索引使用主键作为”指针”而不是使用地址值作为指针的好处是，<strong>减少了当出现行移动或者数据页分裂时辅助索引的维护工作</strong>。也就是聚簇索引这棵树由于页分裂导致的树的结构的调整，不会牵扯到辅助索引这个树的调整。</li>
<li><strong>可以把相关数据保存在一起</strong>。例如实现电子邮箱时，可以根据用户 ID 来聚集数据，这样只需要从磁盘读取少数的数据页就能获取某个用户的全部邮件。如果没有使用聚簇索引，则每封邮件都可能导致一次磁盘 I&#x2F;O。</li>
</ul>
<h3 id="劣势"><a href="#劣势" class="headerlink" title="劣势"></a>劣势</h3><ol>
<li><p><strong>维护索引很昂贵，特别是插入新行或者主键被更新导至要分页(page split)的时候</strong>。建议在大量插入新行后，选在负载较低的时间段，通过OPTIMIZE TABLE优化表，因为必须被移动的行数据可能造成碎片。使用独享表空间可以弱化碎片。</p>
</li>
<li><p>如果主键比较大的话，那辅助索引将会变的更大，因为<strong>辅助索引的叶子存储的是主键值；过长的主键值，会导致非叶子节点占用占用更多的物理空间</strong>。</p>
</li>
</ol>
<h1 id="使用索引的一些注意点"><a href="#使用索引的一些注意点" class="headerlink" title="使用索引的一些注意点"></a>使用索引的一些注意点</h1><h2 id="最左匹配原则"><a href="#最左匹配原则" class="headerlink" title="最左匹配原则"></a>最左匹配原则</h2><p><strong>组合索引的最左前缀匹配原则：使用组合索引查询时，mysql会一直向右匹配直至遇到范围查询(&gt;、&lt;、between、like)就停止匹配。</strong></p>
<p>最左前缀匹配原则和联合索引的<strong>索引存储结构和检索方式</strong>是有关系的。</p>
<p>说白了，就是B树的节点的键进行比较的时候比较原则。</p>
<p>比方说，创建索引了<code>index_abc(a,b,c)</code>，</p>
<p>在组合索引树中，最底层的叶子节点按照第一列按a字段从左到右递增排列，但是b列和c列是相对无序的——只有在a相同的时候，b才是有序的。同理，c列只能在a，b两列相等的情况下小范围内递增有序。</p>
<h2 id="覆盖索引"><a href="#覆盖索引" class="headerlink" title="覆盖索引"></a>覆盖索引</h2><p>覆盖索引并不是说是索引结构，<strong>覆盖索引是一种很常用的优化手段</strong>。</p>
<p>因为在使用辅助索引的时候，我们只可以拿<strong>到主键值和建立的辅助索引列</strong>，相当于获取数据还需要再根据主键查询主键索引再获取到数据。但是试想下这么一种情况，，那是不是意味着我们查询到组合索引的叶子节点就可以直接返回了，而<strong>不需要回表</strong>。这种情况就是覆盖索引。</p>
<p>如何知道是否使用了覆盖索引呢？</p>
<p>explain执行计划，查询Extra列，如果是Using index，表明确实使用了覆盖索引，或者说没有回表操作。</p>
<h2 id="合理建立联合索引"><a href="#合理建立联合索引" class="headerlink" title="合理建立联合索引"></a>合理建立联合索引</h2><p>联合索引，在建立索引的时候，尽量在多个单列索引上判断下是否可以使用联合索引。联合索引的使用<strong>不仅可以节省空间</strong>，还可以更容易的使用到索引覆盖。</p>
<p>比如联合索引（a_b_c），是不是等于有了索引：a，a_b，a_b_c三个索引，这样是不是节省了空间，当然节省的空间并不是三倍于（a，a_b，a_b_c）三个索引。</p>
<p>总之：</p>
<ol>
<li>考虑当前是否已经存在多个可以合并的单列索引，如果有，那么<strong>将当前多个单列索引创建为一个联合索引</strong>。</li>
<li>当前索引<strong>存在频繁使用作为返回字段的列</strong>，这个时候就可以考虑当前列是否可以加入到当前已经存在索引上，使其查询语句可以使用到覆盖索引。</li>
</ol>
<h1 id="索引的创建、删除、使用"><a href="#索引的创建、删除、使用" class="headerlink" title="索引的创建、删除、使用"></a>索引的创建、删除、使用</h1><h2 id="索引的创建"><a href="#索引的创建" class="headerlink" title="索引的创建"></a>索引的创建</h2><ul>
<li><p>建表时创建</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">create</span> <span class="keyword">table</span> 表名（</span><br><span class="line">......</span><br><span class="line">[<span class="keyword">UNIQUE</span><span class="operator">|</span>FULLTEXT<span class="operator">|</span>SPATIAL] INDEX<span class="operator">|</span>KEY </span><br><span class="line">[索引名](字段名 [长度] [<span class="keyword">ASC</span> <span class="operator">|</span> <span class="keyword">DESC</span>]...) [<span class="keyword">USING</span> &#123;BTREE<span class="operator">|</span>HASH&#125;]</span><br><span class="line">）</span><br></pre></td></tr></table></figure>



<ul>
<li>UNIQUE: 可选。表示索引为唯一性索引；FULLTEXT:可选。表示索引为全文索引；SPATIAL:可选。表示索引为空间索引。</li>
<li>INDEX和KEY: 用于指定字段为索引，两者选择其中之一就可以了，作用是一样的。</li>
<li>索引名: 可选。给创建的索引取一个新名称。</li>
<li>字段名: 指定索引对应的字段的名称，该字段必须是前面定义好的字段。</li>
<li>长度: 可选。指索引的长度，必须是字符串类型才可以使用。</li>
<li>ASC: 可选。表示升序排列、DESC:可选。表示降序排列。<br>注：索引方法默认使用B+TREE。</li>
</ul>
</li>
<li><p>建表后创建</p>
<p>第一种方法：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ALTER TABLE 表名 ADD [UNIQUE|FULLTEXT|SPATIAL] INDEX|KEY [索引名] (字段名[(长度)] [ASC|DESC]) [USING 索引方法]；</span><br></pre></td></tr></table></figure>

<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">ALTER</span> <span class="keyword">TABLE</span> mytable <span class="keyword">ADD</span> INDEX idx_c1c2c3 (mycol1, mycol2, mycol3);</span><br></pre></td></tr></table></figure>



<p>第二种方法:</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">CREATE [UNIQUE|FULLTEXT|SPATIAL] INDEX 索引名 ON 表名(字段名) [USING 索引方法]；</span><br></pre></td></tr></table></figure>

<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">CREATE</span> INDEX idx_c1c2c3 <span class="keyword">ON</span> mytable(mycol1, mycol2, mycol3);</span><br></pre></td></tr></table></figure></li>
</ul>
<h2 id="查看与删除索引"><a href="#查看与删除索引" class="headerlink" title="查看与删除索引"></a>查看与删除索引</h2><ul>
<li>查看<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">show</span> index <span class="keyword">from</span> 表名;</span><br></pre></td></tr></table></figure></li>
<li>删除<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">DROP</span> INDEX 索引名 <span class="keyword">ON</span> 表名</span><br><span class="line"><span class="keyword">ALTER</span> <span class="keyword">TABLE</span> 表名 <span class="keyword">DROP</span> INDEX 索引名</span><br></pre></td></tr></table></figure></li>
</ul>
<h1 id="有关索引的补充了解"><a href="#有关索引的补充了解" class="headerlink" title="有关索引的补充了解"></a>有关索引的补充了解</h1><h2 id="自适应哈希索引"><a href="#自适应哈希索引" class="headerlink" title="自适应哈希索引"></a>自适应哈希索引</h2><p><strong>自适应hash索引（Adaptive Hash Index）</strong><br>是InnoDB存储引擎中的内存结构的组成部分。</p>
<p><strong>InnoDB存储引擎会自动根据访问的频率和模式来自动的为某些热点数据建立hash索引</strong>。</p>
<p>InnoDB存储引擎会监控对表上各索引页的查询，如果观察到建立hash索引可以提高查询速度，则自动建立hash索引。这就是自适应哈希索引（Adaptive Hash Index，AHI）<br><strong>AHI是通过缓存池的B+树页构造而来</strong>，因此建立的速度很快，而且不需要对整张表构建hash索引。</p>
<h2 id="索引下推（IPC）"><a href="#索引下推（IPC）" class="headerlink" title="索引下推（IPC）"></a>索引下推（IPC）</h2><p>索引条件下推 (ICP) 是 MySQL 5.6.2 里程碑版本中的新优化器功能之一。</p>
<p><strong>Index Condition Pushdown 的目标是将尽可能多的条件处理（主要是 WHERE 子句）从服务器层转移到存储引擎层</strong>。</p>
<p>举个具体的例子吧，</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">CREATE</span> <span class="keyword">TABLE</span> person (</span><br><span class="line">  personid <span class="type">INTEGER</span> <span class="keyword">PRIMARY</span> KEY,</span><br><span class="line">  firstname <span class="type">CHAR</span>(<span class="number">20</span>),</span><br><span class="line">  lastname <span class="type">CHAR</span>(<span class="number">20</span>),</span><br><span class="line">  postalcode <span class="type">INTEGER</span>,</span><br><span class="line">  age <span class="type">INTEGER</span>,</span><br><span class="line">  address <span class="type">CHAR</span>(<span class="number">50</span>),</span><br><span class="line">  KEY k1（postalcode，age）</span><br><span class="line"> ）ENGINE<span class="operator">=</span>InnoDB;</span><br></pre></td></tr></table></figure>



<p>如果有下面这条sql语句</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">SELECT</span> lastname, firstname <span class="keyword">FROM</span> person</span><br><span class="line"><span class="keyword">WHERE</span> postalcode <span class="keyword">BETWEEN</span> <span class="number">5000</span> <span class="keyword">AND</span> <span class="number">5500</span> <span class="keyword">AND</span> age <span class="keyword">BETWEEN</span> <span class="number">21</span> <span class="keyword">AND</span> <span class="number">22</span>;</span><br></pre></td></tr></table></figure>

<p>根据最左匹配原则，只能使用到索引k1的postalcode列，age列无法使用（因为postalcode是一个范围查询）</p>
<p>如果，没有IPC, 那么怎么处理这条sql呢？</p>
<p>服务器层将sql推给存储引擎层，引擎层只能使用索引的第一个列查询一部分数据（比如说由10000条），</p>
<p>然后将这10000条数据返回给存储引擎层，然后服务器层再根据第二个条件进行过滤，最终留下1000条数据。</p>
<p>有了IPC之后呢，服务器层可以直接把where条件下推给存储引擎层，然后存储引擎可以直接查出这1000条数据，然后再返回给服务器层。</p>
<blockquote>
<p>参考链接</p>
</blockquote>
<p><a target="_blank" rel="noopener" href="http://olavsandstaa.blogspot.com/2011/04/mysql-56-index-condition-pushdown.html">MySQL 5.6：索引条件下推</a></p>
<blockquote>
<p>参考链接</p>
<p><a target="_blank" rel="noopener" href="https://blog.csdn.net/qq_35190492/article/details/109257302">一文搞懂MySQL索引所有知识点（建议收藏）</a></p>
<p><a target="_blank" rel="noopener" href="https://draveness.me/whys-the-design-mysql-b-plus-tree/">为什么 MySQL 使用 B+ 树</a></p>
<p><a target="_blank" rel="noopener" href="https://www.jianshu.com/p/fa8192853184">聚簇索引与非聚簇索引</a></p>
</blockquote>

      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/25/SpringBoot%E6%95%B4%E5%90%88Kafka/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/25/SpringBoot%E6%95%B4%E5%90%88Kafka/" class="post-title-link" itemprop="url">SpringBoot整合Kafka收发消息</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>

      <time title="创建时间：2021-10-25 09:58:57" itemprop="dateCreated datePublished" datetime="2021-10-25T09:58:57+08:00">2021-10-25</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar-check"></i>
      </span>
      <span class="post-meta-item-text">更新于</span>
      <time title="修改时间：2023-01-09 03:12:19" itemprop="dateModified" datetime="2023-01-09T03:12:19+08:00">2023-01-09</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/%E6%B6%88%E6%81%AF%E4%B8%AD%E9%97%B4%E4%BB%B6/" itemprop="url" rel="index"><span itemprop="name">消息中间件</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h1 id="Maven依赖"><a href="#Maven依赖" class="headerlink" title="Maven依赖"></a>Maven依赖</h1><figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.springframework.kafka<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>spring-kafka<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- kafka 流处理相关 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.apache.kafka<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>kafka-streams<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 本地嵌入式kafka，测试用，可无需依赖 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.springframework.kafka<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>spring-kafka-test<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">scope</span>&gt;</span>test<span class="tag">&lt;/<span class="name">scope</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br></pre></td></tr></table></figure>




<h1 id="配置主题"><a href="#配置主题" class="headerlink" title="配置主题"></a>配置主题</h1><p>使用SpringBoot,  一个默认配置的 KafkaAdmin 会被注入，可以直接使用。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@Bean</span></span><br><span class="line"><span class="keyword">public</span> KafkaAdmin <span class="title function_">admin</span><span class="params">()</span> &#123;</span><br><span class="line">    Map&lt;String, Object&gt; configs = <span class="keyword">new</span> <span class="title class_">HashMap</span>&lt;&gt;();</span><br><span class="line">    <span class="comment">// configs.put(AdminClientConfig.BOOTSTRAPx_SERVERS_CONFIG, ...); </span></span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">new</span> <span class="title class_">KafkaAdmin</span>(configs);</span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<p>使用<code>TopicBuilder</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@Bean</span></span><br><span class="line"><span class="keyword">public</span> NewTopic <span class="title function_">topic1</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="keyword">return</span> TopicBuilder.name(<span class="string">&quot;topic1&quot;</span>)  <span class="comment">// 主题名</span></span><br><span class="line">            .partitions(<span class="number">10</span>) <span class="comment">// 分区数</span></span><br><span class="line">            .replicas(<span class="number">3</span>)    <span class="comment">// 备份数</span></span><br><span class="line">            .compact()</span><br><span class="line">            .build();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>



<h1 id="发送消息"><a href="#发送消息" class="headerlink" title="发送消息"></a>发送消息</h1><h2 id="使用KafkaTemplate"><a href="#使用KafkaTemplate" class="headerlink" title="使用KafkaTemplate"></a>使用KafkaTemplate</h2><p>使用<code>KafkaTemplate</code>来发送消息, API</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 1. 需要设置默认的主题： public void setDefaultTopic(String defaultTopic)</span></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">sendDefault</span><span class="params">(V data)</span>;</span><br><span class="line"></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">sendDefault</span><span class="params">(K key, V data)</span>;</span><br><span class="line"></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">sendDefault</span><span class="params">(Integer partition, K key, V data)</span>;</span><br><span class="line"></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">sendDefault</span><span class="params">(Integer partition, Long timestamp, K key, V data)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 2.必须指定主题、值（数据）</span></span><br><span class="line"><span class="comment">// 可以指定分区、键、时间戳</span></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">send</span><span class="params">(String topic, V data)</span>;</span><br><span class="line"></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">send</span><span class="params">(String topic, K key, V data)</span>;</span><br><span class="line"></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">send</span><span class="params">(String topic, Integer partition, K key, V data)</span>;</span><br><span class="line"></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">send</span><span class="params">(String topic, Integer partition, Long timestamp, K key, V data)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 3.使用 kafka-clients 提供的 ProducerRecord </span></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">send</span><span class="params">(ProducerRecord&lt;K, V&gt; record)</span>;</span><br><span class="line"><span class="comment">// 4.使用 spring 封装提供的 message</span></span><br><span class="line">ListenableFuture&lt;SendResult&lt;K, V&gt;&gt; <span class="title function_">send</span><span class="params">(Message&lt;?&gt; message)</span>;</span><br></pre></td></tr></table></figure>

<p>返回值值得注意,<code>ListenableFuture</code>是spring提供的拓展了juc下<code>Future</code>接口的增加了添加了回调功能增强版Future。</p>
<h2 id="发送模式"><a href="#发送模式" class="headerlink" title="发送模式"></a>发送模式</h2><h3 id="fire-and-forget（发完即忘）"><a href="#fire-and-forget（发完即忘）" class="headerlink" title="fire-and-forget（发完即忘）"></a>fire-and-forget（发完即忘）</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kafkaTemplate.send(...)</span><br></pre></td></tr></table></figure>

<p>也就是发完之后完全不做任何处理，发送失败也无所谓，比如传送一些日志的时候。</p>
<h3 id="阻塞"><a href="#阻塞" class="headerlink" title="阻塞"></a>阻塞</h3><p>因为返回的是 <code>ListenableFuture</code>，所以可以阻塞式的等待结果。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 一直阻塞</span></span><br><span class="line">kafkaTemplate.send().get();</span><br><span class="line"><span class="comment">// 等待固定时间</span></span><br><span class="line">kafkaTemplate.send().get(<span class="number">10</span>, TimeUnit.SECONDS);</span><br></pre></td></tr></table></figure>



<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">sendToKafka</span><span class="params">(<span class="keyword">final</span> MyOutputData data)</span> &#123;</span><br><span class="line">    <span class="keyword">final</span> ProducerRecord&lt;String, String&gt; record = createRecord(data);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">try</span> &#123;</span><br><span class="line">        template.send(record).get(<span class="number">10</span>, TimeUnit.SECONDS);</span><br><span class="line">        handleSuccess(data);</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">catch</span> (ExecutionException e) &#123;</span><br><span class="line">        handleFailure(data, record, e.getCause());</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">catch</span> (TimeoutException | InterruptedException e) &#123;</span><br><span class="line">        handleFailure(data, record, e);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>



<h3 id="非阻塞"><a href="#非阻塞" class="headerlink" title="非阻塞"></a>非阻塞</h3><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">sendToKafka</span><span class="params">(<span class="keyword">final</span> MyOutputData data)</span> &#123;</span><br><span class="line">    <span class="keyword">final</span> ProducerRecord&lt;String, String&gt; record = createRecord(data);</span><br><span class="line"></span><br><span class="line">    ListenableFuture&lt;SendResult&lt;Integer, String&gt;&gt; future = template.send(record);</span><br><span class="line">    <span class="comment">// 添加处理回调</span></span><br><span class="line">    future.addCallback(<span class="keyword">new</span> <span class="title class_">KafkaSendCallback</span>&lt;SendResult&lt;Integer, String&gt;&gt;() &#123;</span><br><span class="line"></span><br><span class="line">        <span class="meta">@Override</span></span><br><span class="line">        <span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">onSuccess</span><span class="params">(SendResult&lt;Integer, String&gt; result)</span> &#123;</span><br><span class="line">            handleSuccess(data);</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="meta">@Override</span></span><br><span class="line">        <span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">onFailure</span><span class="params">(KafkaProducerException ex)</span> &#123;</span><br><span class="line">            handleFailure(data, record, ex);</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    &#125;);</span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<p>当然，还有种粗粒度的添加回调，在kafkaTemplate上添加回调：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// LoggingProducerListener，它会记录错误并且在发送成功时不执行任何操作。</span></span><br><span class="line">kafkaTemplate.setProducerListener(<span class="keyword">new</span> <span class="title class_">LoggingProducerListener</span>&lt;&gt;());</span><br><span class="line">kafkaTemplate.setProducerListener(<span class="keyword">new</span> <span class="title class_">ProducerListener</span>&lt;String, String&gt;() &#123;</span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">onSuccess</span><span class="params">(ProducerRecord&lt;String, String&gt; producerRecord, RecordMetadata recordMetadata)</span> &#123;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">onError</span><span class="params">(ProducerRecord&lt;String, String&gt; producerRecord, RecordMetadata recordMetadata, Exception exception)</span> &#123;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line">&#125;);</span><br></pre></td></tr></table></figure>



<h3 id="使用ReplyingKafkaTemplate"><a href="#使用ReplyingKafkaTemplate" class="headerlink" title="使用ReplyingKafkaTemplate"></a>使用ReplyingKafkaTemplate</h3><p>不过需要注意，这个<code>ReplyingKafkaTemplate</code>SpringBoot并没有注入，需要自行构造。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@Bean</span></span><br><span class="line"><span class="keyword">public</span> ReplyingKafkaTemplate&lt;String, String, String&gt; <span class="title function_">replyingTemplate</span><span class="params">(</span></span><br><span class="line"><span class="params">        ProducerFactory&lt;String, String&gt; pf,</span></span><br><span class="line"><span class="params">        ConcurrentMessageListenerContainer&lt;String, String&gt; repliesContainer)</span> &#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">new</span> <span class="title class_">ReplyingKafkaTemplate</span>&lt;&gt;(pf, repliesContainer);</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="meta">@Bean</span></span><br><span class="line"><span class="keyword">public</span> ConcurrentMessageListenerContainer&lt;String, String&gt; <span class="title function_">repliesContainer</span><span class="params">(</span></span><br><span class="line"><span class="params">        ConcurrentKafkaListenerContainerFactory&lt;String, String&gt; containerFactory)</span> &#123;</span><br><span class="line"></span><br><span class="line">    ConcurrentMessageListenerContainer&lt;String, String&gt; repliesContainer =</span><br><span class="line">            containerFactory.createContainer(<span class="string">&quot;replies&quot;</span>);</span><br><span class="line">    repliesContainer.getContainerProperties().setGroupId(<span class="string">&quot;repliesGroup&quot;</span>);</span><br><span class="line">    repliesContainer.setAutoStartup(<span class="literal">false</span>);</span><br><span class="line">    <span class="keyword">return</span> repliesContainer;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>



<p>API</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">RequestReplyFuture&lt;K, V, R&gt; <span class="title function_">sendAndReceive</span><span class="params">(ProducerRecord&lt;K, V&gt; record)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// replyTimeout为等待回复的超时时间</span></span><br><span class="line">RequestReplyFuture&lt;K, V, R&gt; <span class="title function_">sendAndReceive</span><span class="params">(ProducerRecord&lt;K, V&gt; record,</span></span><br><span class="line"><span class="params">    Duration replyTimeout)</span>;</span><br></pre></td></tr></table></figure>



<p>如何使用这个返回值<code>RequestReplyFuture</code>呢？</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">RequestReplyFuture&lt;String, String, String&gt; replyFuture = template.sendAndReceive(record);</span><br><span class="line"><span class="comment">// 发送结果</span></span><br><span class="line">SendResult&lt;String, String&gt; sendResult = replyFuture.getSendFuture().get();</span><br><span class="line">System.out.println(<span class="string">&quot;Sent ok: &quot;</span> + sendResult.getRecordMetadata());</span><br><span class="line"><span class="comment">// 返回的结果</span></span><br><span class="line">ConsumerRecord&lt;String, String&gt; consumerRecord = replyFuture.get();</span><br><span class="line">System.out.println(<span class="string">&quot;Return value: &quot;</span> + consumerRecord.value()); <span class="comment">// 返回值</span></span><br></pre></td></tr></table></figure>



<h2 id="接受消息"><a href="#接受消息" class="headerlink" title="接受消息"></a>接受消息</h2><p>大的方向有两种接受消息的方式,一个low-level一点，一个high-level一点。</p>
<ol>
<li>配置<code>MessageListenerContainer</code>并提供<code>MessageListener</code>。</li>
<li>使用<code>@KafkaListener</code>。</li>
</ol>
<p>其中第二种方法类似于其他MQ的监听器，如<code>@RabbitListener</code>,使用简单方便。</p>
<p>第一种方法相当于SpringBoot提供了一个消息监听器的容器，这个容器负责<code>kafkaConsumer.poll</code>调用，</p>
<p>以及MessageListener的方法调用，主要的动能就是让用户无需关注多线程并发代码的编写，着力于消息的消费与处理。</p>
<p>值得注意的是，即使你没有直接使用第一种方法，<code>@KafkaListener</code>所注解的类也被封装成<code>MessagingMessageListenerAdapter</code>然后由<code>MessageListenerContainer</code>调度。</p>
<p>下面主要介绍@KafkaListener的使用。</p>
<blockquote>
<p>只列出最常用的属性</p>
</blockquote>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">public</span> <span class="meta">@interface</span> KafkaListener &#123;</span><br><span class="line">  </span><br><span class="line">  <span class="comment">// 订阅的主题</span></span><br><span class="line">	String[] topics() <span class="keyword">default</span> &#123;&#125;;</span><br><span class="line"></span><br><span class="line">  <span class="comment">// 用于匹配主题的pattern</span></span><br><span class="line">	String <span class="title function_">topicPattern</span><span class="params">()</span> <span class="keyword">default</span> <span class="string">&quot;&quot;</span>;</span><br><span class="line"></span><br><span class="line">  <span class="comment">// 主题分区</span></span><br><span class="line">	TopicPartition[] topicPartitions() <span class="keyword">default</span> &#123;&#125;;</span><br><span class="line"></span><br><span class="line">	String <span class="title function_">errorHandler</span><span class="params">()</span> <span class="keyword">default</span> <span class="string">&quot;&quot;</span>;</span><br><span class="line"></span><br><span class="line">  <span class="comment">// 消费组id</span></span><br><span class="line">	String <span class="title function_">groupId</span><span class="params">()</span> <span class="keyword">default</span> <span class="string">&quot;&quot;</span>;</span><br><span class="line"></span><br><span class="line">  <span class="comment">// 线程数</span></span><br><span class="line">	String <span class="title function_">concurrency</span><span class="params">()</span> <span class="keyword">default</span> <span class="string">&quot;&quot;</span>;</span><br><span class="line">  </span><br><span class="line">  String[] properties() <span class="keyword">default</span> &#123;&#125;;</span><br><span class="line">  </span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="meta">@interface</span> TopicPartition &#123;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">	String <span class="title function_">topic</span><span class="params">()</span>;</span><br><span class="line"></span><br><span class="line">	String[] partitions() <span class="keyword">default</span> &#123;&#125;;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">	PartitionOffset[] partitionOffsets() <span class="keyword">default</span> &#123;&#125;;</span><br><span class="line"></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="meta">@interface</span> PartitionOffset &#123;</span><br><span class="line"></span><br><span class="line">	String <span class="title function_">partition</span><span class="params">()</span>;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">	String <span class="title function_">initialOffset</span><span class="params">()</span>;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">	String <span class="title function_">relativeToCurrent</span><span class="params">()</span> <span class="keyword">default</span> <span class="string">&quot;false&quot;</span>;</span><br><span class="line"></span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>
















      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/23/Elasticsearch%E5%85%A5%E9%97%A8/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/23/Elasticsearch%E5%85%A5%E9%97%A8/" class="post-title-link" itemprop="url">Elasticsearch入门</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>

      <time title="创建时间：2021-10-23 20:29:14" itemprop="dateCreated datePublished" datetime="2021-10-23T20:29:14+08:00">2021-10-23</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar-check"></i>
      </span>
      <span class="post-meta-item-text">更新于</span>
      <time title="修改时间：2021-10-24 14:24:43" itemprop="dateModified" datetime="2021-10-24T14:24:43+08:00">2021-10-24</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/Elasticsearch/" itemprop="url" rel="index"><span itemprop="name">Elasticsearch</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h1 id="背景知识"><a href="#背景知识" class="headerlink" title="背景知识"></a>背景知识</h1><h2 id="Elaticsearch-Stack-简介"><a href="#Elaticsearch-Stack-简介" class="headerlink" title="Elaticsearch Stack 简介"></a>Elaticsearch Stack 简介</h2><p>Elaticsearch，简称为es， es是一个高扩展的、<strong>分布式</strong>的、<strong>RESTful 风格</strong>的<strong>搜索和数据分析引擎</strong>。</p>
<p>它可以近乎实时的存储、检索数据。本身扩展性很好，可以扩展到上百台服务器，处理PB级别的数据。</p>
<p>Es也使用<strong>Java开发</strong>并使用Lucene作为其核心来实现所有索引和搜索的功能，但是它的目的是通过简单的RESTful API来隐藏Lucene的复杂性，从而让全文搜索变得简单。</p>
<h2 id="Kibana简介"><a href="#Kibana简介" class="headerlink" title="Kibana简介"></a>Kibana简介</h2><p>Kibana 是一个免费且开放的用户界面，能够让您对 Elasticsearch 数据进行可视化，并让您在 Elastic Stack 中进行导航。您可以进行各种操作，从跟踪查询负载，到理解请求如何流经您的整个应用，都能轻松完成。</p>
<p>Es技术栈还有其他工具，目前介绍这两个。</p>
<h1 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h1><p>下面是二者下载链接，下载相应的版本即可。</p>
<p><a target="_blank" rel="noopener" href="https://www.elastic.co/cn/downloads/elasticsearch">elasticsearch下载</a></p>
<p><a target="_blank" rel="noopener" href="https://www.elastic.co/cn/downloads/kibana">kibana下载</a></p>
<p>或者对于mac用户直接使用brew安装启动即可（学习环境使用）</p>
<p>(M1的mac)</p>
<ul>
<li>安装 elasticsearch</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">brew install elastic/tap/elasticsearch-full</span><br></pre></td></tr></table></figure>

<ul>
<li>安装 kibana</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">brew install elastic/tap/kibana-full</span><br></pre></td></tr></table></figure>

<p>分别启动。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">brew services start elasticsearch-full</span><br></pre></td></tr></table></figure>


<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">brew services start kibana-full</span><br></pre></td></tr></table></figure>


<p>访问</p>
<p>es的默认端口 9200： <a target="_blank" rel="noopener" href="http://localhost:9200/">http://localhost:9200/</a></p>
<p>kibana的默认端口 5601： <a target="_blank" rel="noopener" href="http://localhost:5601/">http://localhost:5601/</a></p>
<p>注意：9300是tcp通讯端口，集群间和TCPClient都使用该端口，9200是http协议的RESTful接口。</p>
<p>如果正常展示web界面，以及如下文字，说明安装启动一切正常。</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">  <span class="attr">&quot;name&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;SongyangJi-MacBookAir.local&quot;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;cluster_name&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;elasticsearch_jisongyang&quot;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;cluster_uuid&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;wCrPSWgyQnCCCVCr0Hhc1g&quot;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;version&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;number&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;7.14.2&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;build_flavor&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;default&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;build_type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;tar&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;build_hash&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;6bc13727ce758c0e943c3c21653b3da82f627f75&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;build_date&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;2021-09-15T10:18:09.722761972Z&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;build_snapshot&quot;</span> <span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;lucene_version&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;8.9.0&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;minimum_wire_compatibility_version&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;6.8.0&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;minimum_index_compatibility_version&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;6.0.0-beta1&quot;</span></span><br><span class="line">  <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;tagline&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;You Know, for Search&quot;</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>

<h1 id="Elasticsearch相关概念术语"><a href="#Elasticsearch相关概念术语" class="headerlink" title="Elasticsearch相关概念术语"></a>Elasticsearch相关概念术语</h1><h2 id="关系型数据库的比较与联系"><a href="#关系型数据库的比较与联系" class="headerlink" title="关系型数据库的比较与联系"></a>关系型数据库的比较与联系</h2><table>
<thead>
<tr>
<th>数据库实例&#x2F;Es集群</th>
<th>库&#x2F;索引</th>
<th>表&#x2F;类型</th>
<th>行&#x2F;文档</th>
<th>列&#x2F;字段</th>
</tr>
</thead>
<tbody><tr>
<td>RelationalDB</td>
<td>Databases</td>
<td>Tables</td>
<td>Rows</td>
<td>Columns</td>
</tr>
<tr>
<td>Elasticsearch</td>
<td>Indexes</td>
<td>Types</td>
<td>Documents</td>
<td>Fields</td>
</tr>
</tbody></table>
<h2 id="核心概念"><a href="#核心概念" class="headerlink" title="核心概念"></a>核心概念</h2><ul>
<li><p>index (索引)<br>一个索引就是一个拥有几分相似特征的文档的集合。</p>
</li>
<li><p>type（类型）<br>在一个索引中，你可以定义一种或多种类型。一个类型是你的索引的一个逻辑上的分类&#x2F;分区，其语义完全由你来定。通常，会为具有一组共同字段的文档定义一个类型。</p>
</li>
</ul>
<p>不过，明显上述的概念定义是有点模糊的（不像关系型数据库的库表关系那样泾渭分明），所以在后来的版本中就取消了type这个语义。</p>
<ul>
<li><p>document 文档<br>一个文档是一个可被索引的基础信息单元。文档以JSON（Javascript Object Notation）格式来表示，而JSON是一个到处存在的互联网数据交互格式。在一个index，你可以存储任意多的文档。</p>
</li>
<li><p>field 字段</p>
<p>相当于是数据表的字段，对文档数据根据不同属性进行的分类标识，也就是 Json 中的键。</p>
</li>
<li><p>mapping 映射</p>
<p><strong>mapping是处理数据的方式和规则方面做一些限制</strong>，如某个字段的数据类型、默认值、分析器、是否被索引等等，这些都是映射里面可以设置的，其它就是处理es里面数据的一些使用规则设置也叫做映射，按着最优规则处理数据对性能提高很大，因此才需要建立映射，并且需要思考如何建立映射才能对性能更好。</p>
</li>
</ul>
<h1 id="与-Elasticsearch-交互"><a href="#与-Elasticsearch-交互" class="headerlink" title="与 Elasticsearch 交互"></a>与 Elasticsearch 交互</h1><p>你可以</p>
<ul>
<li>使用elasticsearch提供的UI工具<ol>
<li>elasticsearch-head插件</li>
<li>kibana的dev-tools</li>
</ol>
</li>
<li>使用elasticsearch提供的Restful接口直接访问<ol>
<li>cURL</li>
<li>postman</li>
</ol>
</li>
<li>使用elasticsearch提供的API进行访问<br>也就是用程序调api去与es交互，也是程序开发者的最关心的交互方式。</li>
</ul>
<h2 id="curl"><a href="#curl" class="headerlink" title="curl"></a>curl</h2><p>如果使用curl的话</p>
<ul>
<li>语法</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -X&lt;VERB&gt; &#x27;&lt;PROTOCOL&gt;://&lt;HOST&gt;:&lt;PORT&gt;/&lt;PATH&gt;?&lt;QUERY_STRING&gt;&#x27; -d &#x27;&lt;BODY&gt;&#x27;</span><br></pre></td></tr></table></figure>



<p>各字段的含义：</p>
<table>
<thead>
<tr>
<th><code>VERB</code></th>
<th>适当的 HTTP <em>方法</em> 或 <em>谓词</em> : <code>GET</code>、 <code>POST</code>、 <code>PUT</code>、 <code>HEAD</code> 或者 <code>DELETE</code>。</th>
</tr>
</thead>
<tbody><tr>
<td><code>PROTOCOL</code></td>
<td><code>http</code> 或者 <code>https</code>（如果你在 Elasticsearch 前面有一个 <code>https</code> 代理）</td>
</tr>
<tr>
<td><code>HOST</code></td>
<td>Elasticsearch 集群中任意节点的主机名，或者用 <code>localhost</code> 代表本地机器上的节点。</td>
</tr>
<tr>
<td><code>PORT</code></td>
<td>运行 Elasticsearch HTTP 服务的端口号，默认是 <code>9200</code> 。</td>
</tr>
<tr>
<td><code>PATH</code></td>
<td>API 的终端路径（例如 <code>_count</code> 将返回集群中文档数量）。Path 可能包含多个组件，例如：<code>_cluster/stats</code> 和 <code>_nodes/stats/jvm</code> 。</td>
</tr>
<tr>
<td><code>QUERY_STRING</code></td>
<td>任意可选的查询字符串参数 (例如 <code>?pretty</code> 将格式化地输出 JSON 返回值，使其更容易阅读)</td>
</tr>
<tr>
<td><code>BODY</code></td>
<td>一个 JSON 格式的请求体 (如果请求需要的话)</td>
</tr>
</tbody></table>
<h2 id="使用kibana"><a href="#使用kibana" class="headerlink" title="使用kibana"></a>使用kibana</h2><p>在kibana的开发者工具<a target="_blank" rel="noopener" href="http://localhost:5601/app/dev_tools#/console">console</a>。</p>
<p>以下为 <strong>version 7.14.2</strong> 的示范操作。</p>
<h3 id="索引的CRUD"><a href="#索引的CRUD" class="headerlink" title="索引的CRUD"></a>索引的CRUD</h3><h4 id="创建索引"><a href="#创建索引" class="headerlink" title="创建索引"></a>创建索引</h4><figure class="highlight http"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">PUT /problem</span><br></pre></td></tr></table></figure>

<p>返回值</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">  <span class="attr">&quot;acknowledged&quot;</span> <span class="punctuation">:</span> <span class="literal"><span class="keyword">true</span></span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;shards_acknowledged&quot;</span> <span class="punctuation">:</span> <span class="literal"><span class="keyword">true</span></span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;index&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;problem&quot;</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>



<h4 id="查看索引"><a href="#查看索引" class="headerlink" title="查看索引"></a>查看索引</h4><figure class="highlight http"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">GET /problem</span><br></pre></td></tr></table></figure>

<p>只介绍其中几个字段的含义：</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">  <span class="attr">&quot;problem&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;aliases&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="punctuation">&#125;</span><span class="punctuation">,</span> <span class="comment">// 索引的别名</span></span><br><span class="line">    <span class="attr">&quot;mappings&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="punctuation">&#125;</span><span class="punctuation">,</span> <span class="comment">// mapping的规则</span></span><br><span class="line">    <span class="attr">&quot;settings&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span>  </span><br><span class="line">      <span class="attr">&quot;index&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;routing&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;allocation&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;include&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">              <span class="attr">&quot;_tier_preference&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;data_content&quot;</span></span><br><span class="line">            <span class="punctuation">&#125;</span></span><br><span class="line">          <span class="punctuation">&#125;</span></span><br><span class="line">        <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;number_of_shards&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;1&quot;</span><span class="punctuation">,</span>  <span class="comment">// 分片的数量，低版本的默认值是5，现在默认是1</span></span><br><span class="line">        <span class="attr">&quot;provided_name&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;problem&quot;</span><span class="punctuation">,</span> </span><br><span class="line">        <span class="attr">&quot;creation_date&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;1635053207532&quot;</span><span class="punctuation">,</span> <span class="comment">// 创建时间戳</span></span><br><span class="line">        <span class="attr">&quot;number_of_replicas&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;1&quot;</span><span class="punctuation">,</span>  <span class="comment">// 备份的数量</span></span><br><span class="line">        <span class="attr">&quot;uuid&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;HfPT1GNyTTmYAXCSjIVoVA&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;version&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span>  </span><br><span class="line">          <span class="attr">&quot;created&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;7140299&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span></span><br><span class="line">      <span class="punctuation">&#125;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line">  <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>


<h4 id="删除索引"><a href="#删除索引" class="headerlink" title="删除索引"></a>删除索引</h4><figure class="highlight http"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">DELETE /index</span><br></pre></td></tr></table></figure>

<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">  <span class="attr">&quot;acknowledged&quot;</span> <span class="punctuation">:</span> <span class="literal"><span class="keyword">true</span></span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>



<h3 id="文档的CRUD"><a href="#文档的CRUD" class="headerlink" title="文档的CRUD"></a>文档的CRUD</h3><h4 id="创建文档"><a href="#创建文档" class="headerlink" title="创建文档"></a>创建文档</h4><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">POST problem/_doc</span><br><span class="line">&#123;</span><br><span class="line">    &quot;difficult_level&quot;: &quot;简单&quot;,</span><br><span class="line">    &quot;id&quot;: &quot;1486&quot;,</span><br><span class="line">    &quot;name&quot;: &quot;数组异或操作&quot;,</span><br><span class="line">    &quot;passing_rate&quot;: &quot;84.3%&quot;,</span><br><span class="line">    &quot;passing_rate_number&quot;: 0.843,</span><br><span class="line">    &quot;solutions&quot;: 381,</span><br><span class="line">    &quot;uri&quot;: &quot;xor-operation-in-an-array&quot;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">  <span class="attr">&quot;_index&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;problem&quot;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;_type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;_doc&quot;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;SJ7bsHwBtcEm-AFPQqjS&quot;</span><span class="punctuation">,</span> <span class="comment">// 此id便是自动生成的</span></span><br><span class="line">  <span class="attr">&quot;_version&quot;</span> <span class="punctuation">:</span> <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;result&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;created&quot;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;_shards&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;total&quot;</span> <span class="punctuation">:</span> <span class="number">2</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;successful&quot;</span> <span class="punctuation">:</span> <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;failed&quot;</span> <span class="punctuation">:</span> <span class="number">0</span></span><br><span class="line">  <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;_seq_no&quot;</span> <span class="punctuation">:</span> <span class="number">0</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;_primary_term&quot;</span> <span class="punctuation">:</span> <span class="number">1</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>

<p>插入完文档后，可以看看这个index的信息。</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">  <span class="attr">&quot;problem&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;aliases&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="comment">// 发现 mappings 信息多了出来，主要是刚刚插入的json串的键的信息，如类型，名称等等</span></span><br><span class="line">    <span class="attr">&quot;mappings&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span> </span><br><span class="line">      <span class="attr">&quot;properties&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;difficult_level&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;text&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;fields&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;keyword&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">              <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;keyword&quot;</span><span class="punctuation">,</span></span><br><span class="line">              <span class="attr">&quot;ignore_above&quot;</span> <span class="punctuation">:</span> <span class="number">256</span></span><br><span class="line">            <span class="punctuation">&#125;</span></span><br><span class="line">          <span class="punctuation">&#125;</span></span><br><span class="line">        <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;id&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;text&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;fields&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;keyword&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">              <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;keyword&quot;</span><span class="punctuation">,</span></span><br><span class="line">              <span class="attr">&quot;ignore_above&quot;</span> <span class="punctuation">:</span> <span class="number">256</span></span><br><span class="line">            <span class="punctuation">&#125;</span></span><br><span class="line">          <span class="punctuation">&#125;</span></span><br><span class="line">        <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;name&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;text&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;fields&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;keyword&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">              <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;keyword&quot;</span><span class="punctuation">,</span></span><br><span class="line">              <span class="attr">&quot;ignore_above&quot;</span> <span class="punctuation">:</span> <span class="number">256</span></span><br><span class="line">            <span class="punctuation">&#125;</span></span><br><span class="line">          <span class="punctuation">&#125;</span></span><br><span class="line">        <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;passing_rate&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;text&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;fields&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;keyword&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">              <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;keyword&quot;</span><span class="punctuation">,</span></span><br><span class="line">              <span class="attr">&quot;ignore_above&quot;</span> <span class="punctuation">:</span> <span class="number">256</span></span><br><span class="line">            <span class="punctuation">&#125;</span></span><br><span class="line">          <span class="punctuation">&#125;</span></span><br><span class="line">        <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;passing_rate_number&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;float&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;solutions&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;long&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;uri&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;text&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;fields&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;keyword&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">              <span class="attr">&quot;type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;keyword&quot;</span><span class="punctuation">,</span></span><br><span class="line">              <span class="attr">&quot;ignore_above&quot;</span> <span class="punctuation">:</span> <span class="number">256</span></span><br><span class="line">            <span class="punctuation">&#125;</span></span><br><span class="line">          <span class="punctuation">&#125;</span></span><br><span class="line">        <span class="punctuation">&#125;</span></span><br><span class="line">      <span class="punctuation">&#125;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;settings&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">      <span class="attr">&quot;index&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;routing&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;allocation&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;include&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">              <span class="attr">&quot;_tier_preference&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;data_content&quot;</span></span><br><span class="line">            <span class="punctuation">&#125;</span></span><br><span class="line">          <span class="punctuation">&#125;</span></span><br><span class="line">        <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;number_of_shards&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;1&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;provided_name&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;problem&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;creation_date&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;1635053636611&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;number_of_replicas&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;1&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;uuid&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;v12NNzubQ1u9g6x9wN9W0A&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;version&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">          <span class="attr">&quot;created&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;7140299&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span></span><br><span class="line">      <span class="punctuation">&#125;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line">  <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>





<h4 id="查询文档"><a href="#查询文档" class="headerlink" title="查询文档"></a>查询文档</h4><p>搜索API的最基础的形式是没有指定任何查询的空搜索，它简单地返回集群中所有索引下的所有文档：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">GET /_search</span><br></pre></td></tr></table></figure>



<p>这里我们搜索刚才创建的problem的索引</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">GET problem/_search</span><br></pre></td></tr></table></figure>



<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">  <span class="attr">&quot;took&quot;</span> <span class="punctuation">:</span> <span class="number">2</span><span class="punctuation">,</span> <span class="comment">// 执行整个搜索请求耗费了多少毫秒</span></span><br><span class="line">  <span class="attr">&quot;timed_out&quot;</span> <span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span> <span class="comment">// 是否超时, 可以指定 timeout 为 10 或者 10ms（10毫秒），或者 1s（1秒）如，GET /_search?timeout=10ms</span></span><br><span class="line"></span><br><span class="line">  <span class="attr">&quot;_shards&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="comment">// 在查询中参与分片的总数，以及这些分片成功了多少个失败了多少个</span></span><br><span class="line">    <span class="attr">&quot;total&quot;</span> <span class="punctuation">:</span> <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;successful&quot;</span> <span class="punctuation">:</span> <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;skipped&quot;</span> <span class="punctuation">:</span> <span class="number">0</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;failed&quot;</span> <span class="punctuation">:</span> <span class="number">0</span></span><br><span class="line">  <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">  <span class="attr">&quot;hits&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="comment">// 结果中最重要的部分是 hits</span></span><br><span class="line">    <span class="attr">&quot;total&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="comment">// 配到的文档总数</span></span><br><span class="line">      <span class="attr">&quot;value&quot;</span> <span class="punctuation">:</span> <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">      <span class="attr">&quot;relation&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;eq&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;max_score&quot;</span> <span class="punctuation">:</span> <span class="number">1.0</span><span class="punctuation">,</span> <span class="comment">// 查询所匹配文档的 _score 的最大值</span></span><br><span class="line">    <span class="attr">&quot;hits&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">      <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;_index&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;problem&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;_type&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;_doc&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;SJ7bsHwBtcEm-AFPQqjS&quot;</span><span class="punctuation">,</span> <span class="comment">// 自动生成的id</span></span><br><span class="line">        <span class="attr">&quot;_score&quot;</span> <span class="punctuation">:</span> <span class="number">1.0</span><span class="punctuation">,</span> <span class="comment">// 衡量了文档与查询的匹配程度,  返回的文档是按照 _score 降序排列的</span></span><br><span class="line">        <span class="attr">&quot;_source&quot;</span> <span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="comment">// 源文档，是你自己添加的文档</span></span><br><span class="line">          <span class="attr">&quot;difficult_level&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;简单&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;1486&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;name&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;数组异或操作&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;passing_rate&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;84.3%&quot;</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;passing_rate_number&quot;</span> <span class="punctuation">:</span> <span class="number">0.843</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;solutions&quot;</span> <span class="punctuation">:</span> <span class="number">381</span><span class="punctuation">,</span></span><br><span class="line">          <span class="attr">&quot;uri&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;xor-operation-in-an-array&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span></span><br><span class="line">      <span class="punctuation">&#125;</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line">  <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>






<h1 id="相关链接"><a href="#相关链接" class="headerlink" title="相关链接"></a>相关链接</h1><p><a target="_blank" rel="noopener" href="https://www.elastic.co/guide/cn/elasticsearch/guide/current/index.html">Elasticsearch: 权威指南（中文）</a></p>
<p><a target="_blank" rel="noopener" href="https://www.elastic.co/guide/en/elasticsearch/reference/7.15/index.html">Elasticsearch Guide</a></p>
<p><a target="_blank" rel="noopener" href="https://docs.spring.io/spring-data/elasticsearch/docs/current/reference/html/#reference">Spring 整合 Elasticsearch</a></p>

      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/23/Kafka%E5%88%9D%E6%8E%A2/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/23/Kafka%E5%88%9D%E6%8E%A2/" class="post-title-link" itemprop="url">Kafka初探</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>

      <time title="创建时间：2021-10-23 09:58:57" itemprop="dateCreated datePublished" datetime="2021-10-23T09:58:57+08:00">2021-10-23</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar-check"></i>
      </span>
      <span class="post-meta-item-text">更新于</span>
      <time title="修改时间：2023-01-09 02:13:50" itemprop="dateModified" datetime="2023-01-09T02:13:50+08:00">2023-01-09</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/%E6%B6%88%E6%81%AF%E4%B8%AD%E9%97%B4%E4%BB%B6/" itemprop="url" rel="index"><span itemprop="name">消息中间件</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h1 id="一些网站"><a href="#一些网站" class="headerlink" title="一些网站"></a>一些网站</h1><p><a target="_blank" rel="noopener" href="https://kafka.apache.org/">官网</a></p>
<p><a target="_blank" rel="noopener" href="https://learn.lianglianglee.com/%E4%B8%93%E6%A0%8F/Kafka%E6%A0%B8%E5%BF%83%E6%8A%80%E6%9C%AF%E4%B8%8E%E5%AE%9E%E6%88%98/">Kafka核心技术与实战</a></p>
<p><a target="_blank" rel="noopener" href="http://www.jasongj.com/tags/Kafka/">http://www.jasongj.com/tags/Kafka/</a></p>
<h1 id="安装、测试"><a href="#安装、测试" class="headerlink" title="安装、测试"></a>安装、测试</h1><p><a target="_blank" rel="noopener" href="https://kafka.apache.org/quickstart">https://kafka.apache.org/quickstart</a></p>
<h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2><p><a target="_blank" rel="noopener" href="https://www.apache.org/dyn/closer.cgi?path=/kafka/3.0.0/kafka_2.13-3.0.0.tgz">下载地址</a></p>
<p>配置相关环境。</p>
<h2 id="测试"><a href="#测试" class="headerlink" title="测试"></a>测试</h2><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">cd</span> <span class="variable">$KAFKA_HOME</span></span><br></pre></td></tr></table></figure>



<p>kafka依赖zookeeper，所以要先启动它。(客户端默认连接端口2181)</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">sudo bin/zookeeper-server-start.sh config/zookeeper.properties</span><br></pre></td></tr></table></figure>


<ul>
<li>启动服务器</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># front start</span></span><br><span class="line">sudo bin/kafka-server-start.sh config/server.properties</span><br><span class="line">sudo bin/kafka-server-start.sh -daemon config/server.properties</span><br></pre></td></tr></table></figure>



<ul>
<li>重启服务器</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">sudo bin/kafka-server-stop.sh</span><br><span class="line">sudo bin/kafka-server-start.sh -daemon config/server.properties</span><br></pre></td></tr></table></figure>





<ul>
<li>创建topic</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic <span class="built_in">test</span></span><br></pre></td></tr></table></figure>



<ul>
<li>列出topic</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">bin/kafka-topics.sh --list --bootstrap-server localhost:9092</span><br></pre></td></tr></table></figure>



<ul>
<li>查看某个 topic</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">bin/kafka-topics.sh --describe --topic <span class="built_in">test</span> --bootstrap-server localhost:9092</span><br></pre></td></tr></table></figure>



<ul>
<li>发布消息</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">bin/kafka-console-producer.sh --topic <span class="built_in">test</span> --bootstrap-server localhost:9092</span><br></pre></td></tr></table></figure>



<ul>
<li>消费消息</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">bin/kafka-console-consumer.sh --topic <span class="built_in">test</span> --from-beginning --bootstrap-server localhost:9092</span><br></pre></td></tr></table></figure>



<p>能够呈现出来的效果就是，在<strong>发布消息的console</strong>中发布一条消息（enter分割），在<strong>消费消息的console</strong>中就会输出一条消息。</p>
<h2 id="SpringBoot-Quick-Start"><a href="#SpringBoot-Quick-Start" class="headerlink" title="SpringBoot Quick Start"></a>SpringBoot Quick Start</h2><ul>
<li>yml配置</li>
</ul>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">spring:</span></span><br><span class="line">  <span class="attr">kafka:</span></span><br><span class="line">    <span class="attr">bootstrap-servers:</span> <span class="string">localhost:9092</span></span><br><span class="line">    <span class="attr">consumer:</span></span><br><span class="line">      <span class="attr">group-id:</span> <span class="string">foo</span></span><br><span class="line">      <span class="attr">auto-offset-reset:</span> <span class="string">earliest</span></span><br></pre></td></tr></table></figure>





<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@Service</span></span><br><span class="line"><span class="keyword">public</span> <span class="keyword">class</span> <span class="title class_">Demo</span> &#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">final</span> KafkaTemplate&lt;String, String&gt; template;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Autowired</span></span><br><span class="line">    <span class="keyword">public</span> <span class="title function_">Demo</span><span class="params">(KafkaTemplate&lt;String, String&gt; template)</span> &#123;</span><br><span class="line">        <span class="built_in">this</span>.template = template;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">final</span> <span class="type">String</span> <span class="variable">topicName</span> <span class="operator">=</span> <span class="string">&quot;myTopic&quot;</span>;</span><br><span class="line"></span><br><span class="line">    <span class="type">int</span> <span class="variable">idx</span> <span class="operator">=</span> <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Scheduled(fixedRate = 1000)</span></span><br><span class="line">    <span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">publish</span><span class="params">()</span> &#123;</span><br><span class="line">        <span class="keyword">if</span> (idx++ &gt; <span class="number">10</span>) &#123;</span><br><span class="line">            <span class="keyword">return</span>;</span><br><span class="line">        &#125;</span><br><span class="line">        template.send(topicName, String.valueOf(idx));</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@KafkaListener(topics = topicName)</span></span><br><span class="line">    <span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">listen</span><span class="params">(ConsumerRecord&lt;String, String&gt; cr)</span> &#123;</span><br><span class="line">        System.out.println(cr);</span><br><span class="line">        System.out.println();</span><br><span class="line">    &#125;</span><br><span class="line">    </span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>





<p>输出:</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">ConsumerRecord(topic = myTopic, partition = 0, leaderEpoch = 0, offset = 2, CreateTime = 1635125751646, serialized key size = -1, serialized value size = 1, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = 1)</span><br><span class="line"></span><br><span class="line">ConsumerRecord(topic = myTopic, partition = 0, leaderEpoch = 0, offset = 3, CreateTime = 1635125752304, serialized key size = -1, serialized value size = 1, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = 2)</span><br><span class="line"></span><br><span class="line">ConsumerRecord(topic = myTopic, partition = 0, leaderEpoch = 0, offset = 4, CreateTime = 1635125753302, serialized key size = -1, serialized value size = 1, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = 3)</span><br><span class="line"></span><br><span class="line">ConsumerRecord(topic = myTopic, partition = 0, leaderEpoch = 0, offset = 5, CreateTime = 1635125754303, serialized key size = -1, serialized value size = 1, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = 4)</span><br><span class="line"></span><br><span class="line">ConsumerRecord(topic = myTopic, partition = 0, leaderEpoch = 0, offset = 6, CreateTime = 1635125755300, serialize</span><br></pre></td></tr></table></figure>



<p>可以看到消息的各种属性，如主题、分区、偏移量、时间戳、键、值等等。</p>
<h1 id="Kafka-Eagle"><a href="#Kafka-Eagle" class="headerlink" title="Kafka-Eagle"></a>Kafka-Eagle</h1><p>必须配置好</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">ln -s &quot;xxx&quot; /usr/local/jdk8</span><br><span class="line">export JAVA_HOME=/usr/local/jdk8</span><br><span class="line">export KE_HOME=/usr/local/kafka-eagle-bin-3.0.0/efak-web-3.0.0</span><br></pre></td></tr></table></figure>



<p>修改<code>system-config.properties</code></p>
<p>主要修改了<code>efak.zk.cluster.alias=cluster1 cluster1.zk.list=127.0.0.1:2181</code> 还有<code>kafka mysql jdbc driver address</code></p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br></pre></td><td class="code"><pre><span class="line">######################################</span><br><span class="line"># multi zookeeper &amp; kafka cluster list</span><br><span class="line"># Settings prefixed with &#x27;kafka.eagle.&#x27; will be deprecated, use &#x27;efak.&#x27; instead</span><br><span class="line">######################################</span><br><span class="line">#efak.zk.cluster.alias=cluster1,cluster2</span><br><span class="line">#cluster1.zk.list=tdn1:2181,tdn2:2181,tdn3:2181</span><br><span class="line">#cluster2.zk.list=xdn10:2181,xdn11:2181,xdn12:2181</span><br><span class="line"></span><br><span class="line">efak.zk.cluster.alias=cluster1</span><br><span class="line">cluster1.zk.list=127.0.0.1:2181</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># zookeeper enable acl</span><br><span class="line">######################################</span><br><span class="line">cluster1.zk.acl.enable=false</span><br><span class="line">cluster1.zk.acl.schema=digest</span><br><span class="line">cluster1.zk.acl.username=test</span><br><span class="line">cluster1.zk.acl.password=test123</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># broker size online list</span><br><span class="line">######################################</span><br><span class="line">cluster1.efak.broker.size=20</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># zk client thread limit</span><br><span class="line">######################################</span><br><span class="line">kafka.zk.limit.size=16</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># EFAK webui port</span><br><span class="line">######################################</span><br><span class="line">efak.webui.port=8048</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># EFAK enable distributed</span><br><span class="line">######################################</span><br><span class="line">efak.distributed.enable=false</span><br><span class="line">efak.cluster.mode.status=master</span><br><span class="line">efak.worknode.master.host=localhost</span><br><span class="line">efak.worknode.port=8085</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka jmx acl and ssl authenticate</span><br><span class="line">######################################</span><br><span class="line">cluster1.efak.jmx.acl=false</span><br><span class="line">cluster1.efak.jmx.user=keadmin</span><br><span class="line">cluster1.efak.jmx.password=keadmin123</span><br><span class="line">cluster1.efak.jmx.ssl=false</span><br><span class="line">cluster1.efak.jmx.truststore.location=/data/ssl/certificates/kafka.truststore</span><br><span class="line">cluster1.efak.jmx.truststore.password=ke123456</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka offset storage</span><br><span class="line">######################################</span><br><span class="line">cluster1.efak.offset.storage=kafka</span><br><span class="line">cluster2.efak.offset.storage=zk</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka jmx uri</span><br><span class="line">######################################</span><br><span class="line">cluster1.efak.jmx.uri=service:jmx:rmi:///jndi/rmi://%s/jmxrmi</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka metrics, 15 days by default</span><br><span class="line">######################################</span><br><span class="line">efak.metrics.charts=true</span><br><span class="line">efak.metrics.retain=15</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka sql topic records max</span><br><span class="line">######################################</span><br><span class="line">efak.sql.topic.records.max=5000</span><br><span class="line">efak.sql.topic.preview.records.max=10</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># delete kafka topic token</span><br><span class="line">######################################</span><br><span class="line">efak.topic.token=keadmin</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka sasl authenticate</span><br><span class="line">######################################</span><br><span class="line">cluster1.efak.sasl.enable=false</span><br><span class="line">cluster1.efak.sasl.protocol=SASL_PLAINTEXT</span><br><span class="line">cluster1.efak.sasl.mechanism=SCRAM-SHA-256</span><br><span class="line">cluster1.efak.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username=&quot;kafka&quot; password=&quot;kafka-eagle&quot;;</span><br><span class="line">cluster1.efak.sasl.client.id=</span><br><span class="line">cluster1.efak.blacklist.topics=</span><br><span class="line">cluster1.efak.sasl.cgroup.enable=false</span><br><span class="line">cluster1.efak.sasl.cgroup.topics=</span><br><span class="line">cluster2.efak.sasl.enable=false</span><br><span class="line">cluster2.efak.sasl.protocol=SASL_PLAINTEXT</span><br><span class="line">cluster2.efak.sasl.mechanism=PLAIN</span><br><span class="line">cluster2.efak.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=&quot;kafka&quot; password=&quot;kafka-eagle&quot;;</span><br><span class="line">cluster2.efak.sasl.client.id=</span><br><span class="line">cluster2.efak.blacklist.topics=</span><br><span class="line">cluster2.efak.sasl.cgroup.enable=false</span><br><span class="line">cluster2.efak.sasl.cgroup.topics=</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka ssl authenticate</span><br><span class="line">######################################</span><br><span class="line">cluster3.efak.ssl.enable=false</span><br><span class="line">cluster3.efak.ssl.protocol=SSL</span><br><span class="line">cluster3.efak.ssl.truststore.location=</span><br><span class="line">cluster3.efak.ssl.truststore.password=</span><br><span class="line">cluster3.efak.ssl.keystore.location=</span><br><span class="line">cluster3.efak.ssl.keystore.password=</span><br><span class="line">cluster3.efak.ssl.key.password=</span><br><span class="line">cluster3.efak.ssl.endpoint.identification.algorithm=https</span><br><span class="line">cluster3.efak.blacklist.topics=</span><br><span class="line">cluster3.efak.ssl.cgroup.enable=false</span><br><span class="line">cluster3.efak.ssl.cgroup.topics=</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka sqlite jdbc driver address</span><br><span class="line">######################################</span><br><span class="line">#efak.driver=org.sqlite.JDBC</span><br><span class="line">#efak.url=jdbc:sqlite:/hadoop/kafka-eagle/db/ke.db</span><br><span class="line">#efak.username=root</span><br><span class="line">#efak.password=www.kafka-eagle.org</span><br><span class="line"></span><br><span class="line">######################################</span><br><span class="line"># kafka mysql jdbc driver address</span><br><span class="line">######################################</span><br><span class="line">efak.driver=com.mysql.cj.jdbc.Driver</span><br><span class="line">efak.url=jdbc:mysql://127.0.0.1:3306/ke?useUnicode=true&amp;characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull</span><br><span class="line">efak.username=root</span><br><span class="line">efak.password=root</span><br></pre></td></tr></table></figure>





<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">cd $KE_HOME</span><br><span class="line">bin/ke.sh start</span><br></pre></td></tr></table></figure>



<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line">Welcome to</span><br><span class="line">    ______    ______    ___     __ __</span><br><span class="line">   / ____/   / ____/   /   |   / //_/</span><br><span class="line">  / __/     / /_      / /| |  / ,&lt;</span><br><span class="line"> / /___    / __/     / ___ | / /| |</span><br><span class="line">/_____/   /_/       /_/  |_|/_/ |_|</span><br><span class="line">( Eagle For Apache Kafka® )</span><br><span class="line"></span><br><span class="line">Version v3.0.0 -- Copyright 2016-2022</span><br><span class="line">*******************************************************************</span><br><span class="line">* EFAK Service has started success.</span><br><span class="line">* Welcome, Now you can visit &#x27;http://192.168.0.100:8048&#x27;</span><br><span class="line">* Account:admin ,Password:123456</span><br><span class="line">*******************************************************************</span><br><span class="line">* &lt;Usage&gt; ke.sh [start|status|stop|restart|stats] &lt;/Usage&gt;</span><br><span class="line">* &lt;Usage&gt; https://www.kafka-eagle.org/ &lt;/Usage&gt;</span><br><span class="line">*******************************************************************</span><br></pre></td></tr></table></figure>



<p>介绍 <a target="_blank" rel="noopener" href="https://juejin.cn/post/6971224791793532941">https://juejin.cn/post/6971224791793532941</a></p>
<p>官网 <a target="_blank" rel="noopener" href="http://www.kafka-eagle.org/">http://www.kafka-eagle.org/</a></p>
<p>下载 <a target="_blank" rel="noopener" href="https://github.com/smartloli/kafka-eagle-bin/tags">https://github.com/smartloli/kafka-eagle-bin/tags</a></p>
<p>安装 <a target="_blank" rel="noopener" href="http://www.kafka-eagle.org/articles/docs/installation/linux-macos.html">http://www.kafka-eagle.org/articles/docs/installation/linux-macos.html</a></p>

      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/19/Hadoop%E5%85%A5%E9%97%A8/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/19/Hadoop%E5%85%A5%E9%97%A8/" class="post-title-link" itemprop="url">Hadoop入门</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>
      

      <time title="创建时间：2021-10-19 11:31:02 / 修改时间：22:26:18" itemprop="dateCreated datePublished" datetime="2021-10-19T11:31:02+08:00">2021-10-19</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/Hadoop/" itemprop="url" rel="index"><span itemprop="name">Hadoop</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h1 id="背景、概念"><a href="#背景、概念" class="headerlink" title="背景、概念"></a>背景、概念</h1><blockquote>
<p>Apache Hadoop 软件库是一个框架，允许使用简单的编程模型跨计算机集群分布式处理大型数据集。它旨在从单个服务器扩展到数千台机器，每台机器都提供本地计算和存储。该库本身不是依靠硬件来提供高可用性，而是设计用于检测和处理应用层的故障，因此在计算机集群之上提供高可用性服务，每台计算机都可能容易出现故障。</p>
</blockquote>
<h2 id="模块"><a href="#模块" class="headerlink" title="模块"></a>模块</h2><p>该项目包括以下模块：</p>
<ul>
<li><strong>Hadoop Common</strong>：支持其他 Hadoop 模块的通用实用程序。</li>
<li><strong>Hadoop 分布式文件系统 (HDFS™)<strong>：提供对应用程序数据的</strong>高吞吐量访问</strong>的<strong>分布式文件系统</strong>。</li>
<li><strong>Hadoop YARN</strong>：用于<strong>作业调度</strong>和<strong>集群资源管理</strong>的框架。</li>
<li><strong>Hadoop MapReduce</strong>：一个基于 YARN 的系统，用于<strong>并行处理大型数据集</strong>。</li>
</ul>
<p><a target="_blank" rel="noopener" href="https://hadoop.apache.org/">hadoop官网</a></p>

      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/19/ZooKeeper%E5%85%A5%E9%97%A8/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/19/ZooKeeper%E5%85%A5%E9%97%A8/" class="post-title-link" itemprop="url">ZooKeeper入门</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>
      

      <time title="创建时间：2021-10-19 10:04:29 / 修改时间：10:59:06" itemprop="dateCreated datePublished" datetime="2021-10-19T10:04:29+08:00">2021-10-19</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/ZooKeeper/" itemprop="url" rel="index"><span itemprop="name">ZooKeeper</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h1 id="安装、配置"><a href="#安装、配置" class="headerlink" title="安装、配置"></a>安装、配置</h1><h2 id="下载安装"><a href="#下载安装" class="headerlink" title="下载安装"></a>下载安装</h2><p><a target="_blank" rel="noopener" href="https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0-bin.tar.gz">下载链接</a></p>
<p>下载、解压之后，目录结构如下：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">jisongyang@SongyangJi-MacBookAir apache-zookeeper-3.7.0-bin % ls -el</span><br><span class="line">total 48</span><br><span class="line">-rw-r--r--@  1 1000        1000   11358  3 17  2021 LICENSE.txt</span><br><span class="line">-rw-r--r--@  1 1000        1000     432  3 17  2021 NOTICE.txt</span><br><span class="line">-rw-r--r--@  1 1000        1000    2214  3 17  2021 README.md</span><br><span class="line">-rw-r--r--@  1 1000        1000    3570  3 17  2021 README_packaging.md</span><br><span class="line">drwxr-xr-x@ 18 jisongyang  1000     576  8 12 19:55 bin</span><br><span class="line">drwxr-xr-x@  6 1000        1000     192  8 12 03:39 conf</span><br><span class="line">drwxr-xr-x@ 25 1000        1000     800  3 17  2021 docs</span><br><span class="line">drwxr-xr-x  60 root        wheel   1920  8 11 18:41 lib</span><br><span class="line">drwxr-xr-x   5 root        wheel    160  9 12 12:49 logs</span><br></pre></td></tr></table></figure>

<p>照往常一样，配置一下环境变量。</p>
<h2 id="配置"><a href="#配置" class="headerlink" title="配置"></a>配置</h2><p>要启动需要一个配置文件，在 conf 在创建：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">cp zoo_sample.cfg zoo.cfg</span><br></pre></td></tr></table></figure>

<h2 id="单机运行"><a href="#单机运行" class="headerlink" title="单机运行"></a>单机运行</h2><h3 id="启动"><a href="#启动" class="headerlink" title="启动"></a>启动</h3><p>因为我放在了 &#x2F;usr&#x2F;local 目录下，加上sudo。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">sudo zkServer start</span><br><span class="line">````</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_">#</span><span class="language-bash"><span class="comment">## 查看状态</span></span></span><br></pre></td></tr></table></figure>
<p>zkServer status</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">这里的standalone指的是单机模。</span><br><span class="line"></span><br><span class="line">### 关闭</span><br><span class="line">```shell</span><br><span class="line">sudo zkServer stop</span><br></pre></td></tr></table></figure>



<h1 id="客户端相关"><a href="#客户端相关" class="headerlink" title="客户端相关"></a>客户端相关</h1><h2 id="zkCli"><a href="#zkCli" class="headerlink" title="zkCli"></a>zkCli</h2><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">bin/zkCli.sh</span><br></pre></td></tr></table></figure>
<p>指定服务器。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">bin/zkCli.sh -server 127.0.0.1:2181</span><br><span class="line">````</span><br><span class="line"></span><br><span class="line">当然还有其他启动参数，不再一一介绍。</span><br><span class="line"></span><br><span class="line">输入`help`,查看命令：</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<p>[zkshell: 0] help<br>ZooKeeper -server host:port cmd args<br>addauth scheme auth<br>close<br>config [-c] [-w] [-s]<br>connect host:port<br>create [-s] [-e] [-c] [-t ttl] path [data] [acl]<br>delete [-v version] path<br>deleteall path<br>delquota [-n|-b] path<br>get [-s] [-w] path<br>getAcl [-s] path<br>getAllChildrenNumber path<br>getEphemerals path<br>history<br>listquota path<br>ls [-s] [-w] [-R] path<br>ls2 path [watch]<br>printwatches on|off<br>quit<br>reconfig [-s] [-v version] [[-file path] | [-members serverID&#x3D;host:port1:port2;port3[,…]<em>]] | [-add serverId&#x3D;host:port1:port2;port3[,…]]</em> [-remove serverId[,…]*]<br>redo cmdno<br>removewatches path [-c|-d|-a] [-l]<br>rmr path<br>set [-s] [-v version] path data<br>setAcl [-s] [-v version] [-R] path acl<br>setquota -n|-b val path<br>stat [-w] path<br>sync path</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">最简单的如`ls /`命令列出所有节点。</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">## zkui</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">+ 下载</span><br><span class="line"></span><br><span class="line">下载链接：https://github.com/DeemOpen/zkui</span><br><span class="line"></span><br><span class="line">```shell</span><br><span class="line">git clone https://github.com/DeemOpen/zkui.git</span><br></pre></td></tr></table></figure>



<ul>
<li>安装</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">cd zkui/ # 进入工程界面</span><br><span class="line">mvn clean install # 进行maven打包，执行成功后会生成target文件夹，其中有jar文件。</span><br></pre></td></tr></table></figure>



<ul>
<li>配置</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">zkui web页面访问端口</span></span><br><span class="line">serverPort=9090</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">zookeeper集群的IP地址和端口（这里配置为单机模式）</span></span><br><span class="line">zkServer=localhost:2181</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">设置登录zkui的用户名和密码，这里我们将用户名和密码都设置为admin</span></span><br><span class="line">userSet = &#123;&quot;users&quot;: [&#123; &quot;username&quot;:&quot;admin&quot; , &quot;password&quot;:&quot;admin&quot;,&quot;role&quot;: &quot;ADMIN&quot; &#125;,&#123; &quot;username&quot;:&quot;appconfig&quot; , &quot;password&quot;:&quot;appconfig&quot;,&quot;role&quot;: &quot;USER&quot; &#125;]&#125;</span><br></pre></td></tr></table></figure>



<ul>
<li>启动</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">nohup java -jar target/zkui-2.0-SNAPSHOT-jar-with-dependencies.jar &amp;</span><br></pre></td></tr></table></figure>






<h1 id="参考链接"><a href="#参考链接" class="headerlink" title="参考链接"></a>参考链接</h1><p><a target="_blank" rel="noopener" href="https://zookeeper.apache.org/">Zookeeper官网</a></p>
<p><a target="_blank" rel="noopener" href="http://ningg.top/zookeeper-getting-started/">ZooKeeper 初探：安装、使用</a></p>
<p><a target="_blank" rel="noopener" href="https://www.jianshu.com/p/746799d3db07">安装zkui</a></p>

      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




    


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/10/19/MongoDB-%E8%BF%9B%E9%98%B6(%E8%81%9A%E5%90%88)/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="SongyangJi">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="JsyBlog">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
      <meta itemprop="name" content="undefined | JsyBlog">
      <meta itemprop="description" content="">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          <a href="/2021/10/19/MongoDB-%E8%BF%9B%E9%98%B6(%E8%81%9A%E5%90%88)/" class="post-title-link" itemprop="url">MongoDB 进阶(聚合)</a>
        </h2>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>

      <time title="创建时间：2021-10-19 08:57:21" itemprop="dateCreated datePublished" datetime="2021-10-19T08:57:21+08:00">2021-10-19</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar-check"></i>
      </span>
      <span class="post-meta-item-text">更新于</span>
      <time title="修改时间：2021-12-30 03:03:36" itemprop="dateModified" datetime="2021-12-30T03:03:36+08:00">2021-12-30</time>
    </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/NoSQL/" itemprop="url" rel="index"><span itemprop="name">NoSQL</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
          <h1 id="聚合"><a href="#聚合" class="headerlink" title="聚合"></a>聚合</h1><p>什么是聚合管道？</p>
<ul>
<li><strong>聚合管道是基于数据处理管道概念建模的数据聚合框架</strong>。</li>
<li>文档进入<strong>多阶段管道</strong>，将文档转换为聚合结果。</li>
<li>MongoDB 聚合管道由<strong>阶段</strong>组成。每个阶段都会在文档通过管道时对其进行转换。</li>
<li>管道阶段可以在管道中多次出现，但<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/out/#mongodb-pipeline-pipe.-out"><code>$out</code></a>，<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/merge/#mongodb-pipeline-pipe.-merge"><code>$merge</code></a>、 和 <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/geoNear/#mongodb-pipeline-pipe.-geoNear"><code>$geoNear</code></a>阶段除外。（相当于 Java 流式计算的的最后一步归约操作）。</li>
</ul>
<p><strong>语法</strong></p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">db.collection.aggregate( [ &#123; &lt;stage&gt; &#125;, ... ] )</span><br></pre></td></tr></table></figure>



<h2 id="聚合管道的阶段"><a href="#聚合管道的阶段" class="headerlink" title="聚合管道的阶段"></a>聚合管道的阶段</h2><p>这里只列出最常用的，完整请看参考文档。</p>
<table>
<thead>
<tr>
<th align="left">阶段</th>
<th align="left">描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/addFields/#mongodb-pipeline-pipe.-addFields"><code>$addFields</code></a></td>
<td align="left">向文档添加新字段。类似于 <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/project/#mongodb-pipeline-pipe.-project"><code>$project</code></a>，<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/addFields/#mongodb-pipeline-pipe.-addFields"><code>$addFields</code></a>对流中的每个文档进行整形；具体来说，通过向包含输入文档中现有字段和新添加字段的输出文档添加新字段。<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/set/#mongodb-pipeline-pipe.-set"><code>$set</code></a>是 的别名<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/addFields/#mongodb-pipeline-pipe.-addFields"><code>$addFields</code></a>。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/count/#mongodb-pipeline-pipe.-count"><code>$count</code></a></td>
<td align="left">返回聚合管道此阶段的文档数计数。与<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/count-accumulator/#mongodb-group-grp.-count"><code>$count</code></a>聚合累加器不同。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/group/#mongodb-pipeline-pipe.-group"><code>$group</code></a></td>
<td align="left">按指定的标识符表达式对输入文档进行分组，并将累加器表达式（如果指定）应用于每个组。使用所有输入文档并为每个不同的组输出一个文档。输出文档仅包含标识符字段和累积字段（如果指定）。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/limit/#mongodb-pipeline-pipe.-limit"><code>$limit</code></a></td>
<td align="left">将未修改的前<em>n 个</em>文档传递到管道，其中<em>n</em>是指定的限制。对于每个输入文档，输出一个文档（对于前<em>n 个</em>文档）或零个文档（在前<em>n 个</em>文档之后）。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#mongodb-pipeline-pipe.-lookup"><code>$lookup</code></a></td>
<td align="left">对<em>同一</em>数据库中的另一个集合执行左外部 联接，以从“联接”集合中过滤文档以进行处理。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/match/#mongodb-pipeline-pipe.-match"><code>$match</code></a></td>
<td align="left">过滤文档流以只允许匹配的文档未经修改地传递到下一个管道阶段。 <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/match/#mongodb-pipeline-pipe.-match"><code>$match</code></a>使用标准的 MongoDB 查询。对于每个输入文档，输出一个文档（匹配）或零个文档（不匹配）。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/merge/#mongodb-pipeline-pipe.-merge"><code>$merge</code></a></td>
<td align="left">将聚合管道的结果文档写入集合。该阶段可以将（插入新文档、合并文档、替换文档、保留现有文档、操作失败、使用自定义更新管道处理文档）结果合并到输出集合中。要使用该<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/merge/#mongodb-pipeline-pipe.-merge"><code>$merge</code></a>阶段，它必须是管道中的最后一个阶段。<em>4.2版中的新功能</em>。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/out/#mongodb-pipeline-pipe.-out"><code>$out</code></a></td>
<td align="left">将聚合管道的结果文档写入集合。要使用该<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/out/#mongodb-pipeline-pipe.-out"><code>$out</code></a>阶段，它必须是管道中的最后一个阶段。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/planCacheStats/#mongodb-pipeline-pipe.-planCacheStats"><code>$planCacheStats</code></a></td>
<td align="left">返回集合的<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/core/query-plans/">计划缓存</a>信息。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/project/#mongodb-pipeline-pipe.-project"><code>$project</code></a></td>
<td align="left">重塑流中的每个文档，例如通过添加新字段或删除现有字段。对于每个输入文档，输出一个文档。另请参阅<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/unset/#mongodb-pipeline-pipe.-unset"><code>$unset</code></a>删除现有字段。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/replaceWith/#mongodb-pipeline-pipe.-replaceWith"><code>$replaceWith</code></a></td>
<td align="left">用指定的嵌入文档替换文档。该操作替换输入文档中的所有现有字段，包括该<code>_id</code>字段。指定嵌入在输入文档中的文档以将嵌入的文档提升到顶级。<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/replaceWith/#mongodb-pipeline-pipe.-replaceWith"><code>$replaceWith</code></a>是<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/replaceRoot/#mongodb-pipeline-pipe.-replaceRoot"><code>$replaceRoot</code></a>stage的别名 。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/sample/#mongodb-pipeline-pipe.-sample"><code>$sample</code></a></td>
<td align="left">从其输入中随机选择指定数量的文档。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.atlas.mongodb.com/reference/atlas-search/query-syntax/#mongodb-pipeline-pipe.-search"><code>$search</code></a></td>
<td align="left">对<a target="_blank" rel="noopener" href="https://docs.atlas.mongodb.com/reference/atlas-search/query-syntax/">Atlas</a> 集合中的一个或多个字段执行全文搜索 。笔记<code>$search</code> 仅适用于 MongoDB Atlas 集群，不适用于自管理部署。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/set/#mongodb-pipeline-pipe.-set"><code>$set</code></a></td>
<td align="left">向文档添加新字段。类似于 <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/project/#mongodb-pipeline-pipe.-project"><code>$project</code></a>，<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/set/#mongodb-pipeline-pipe.-set"><code>$set</code></a>对流中的每个文档进行整形；具体来说，通过向包含输入文档中现有字段和新添加字段的输出文档添加新字段。<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/set/#mongodb-pipeline-pipe.-set"><code>$set</code></a>是<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/addFields/#mongodb-pipeline-pipe.-addFields"><code>$addFields</code></a>stage的别名。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/skip/#mongodb-pipeline-pipe.-skip"><code>$skip</code></a></td>
<td align="left">跳过前<em>n 个</em>文档，其中<em>n</em>是指定的跳过编号，并将未修改的剩余文档传递到管道。对于每个输入文档，输出零个文档（对于前<em>n 个</em>文档）或一个文档（如果在前<em>n 个</em>文档之后）。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/sort/#mongodb-pipeline-pipe.-sort"><code>$sort</code></a></td>
<td align="left">按指定的排序键对文档流重新排序。只是顺序变了；文件保持不变。对于每个输入文档，输出一个文档。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/unionWith/#mongodb-pipeline-pipe.-unionWith"><code>$unionWith</code></a></td>
<td align="left">执行两个集合的并集；ie 将来自两个集合的管道结果合并为一个结果集。<em>4.4版中的新功能</em>。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/unset/#mongodb-pipeline-pipe.-unset"><code>$unset</code></a></td>
<td align="left">从文档中删除&#x2F;排除字段。<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/unset/#mongodb-pipeline-pipe.-unset"><code>$unset</code></a>是<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/project/#mongodb-pipeline-pipe.-project"><code>$project</code></a>删除字段的阶段的别名。</td>
</tr>
</tbody></table>
<p><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation-pipeline/">参考文档</a></p>
<h2 id="聚合管道表达式"><a href="#聚合管道表达式" class="headerlink" title="聚合管道表达式"></a>聚合管道表达式</h2><blockquote>
<p>详细的 <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/">参考文档</a></p>
</blockquote>
<h1 id="常用例子"><a href="#常用例子" class="headerlink" title="常用例子"></a>常用例子</h1><h2 id="group"><a href="#group" class="headerlink" title="group"></a>group</h2><h3 id="含义与作用"><a href="#含义与作用" class="headerlink" title="含义与作用"></a>含义与作用</h3><p>按指定的<code>_id</code>表达式对输入文档进行分组，并为每个不同的分组输出一个文档。<code>_id</code>每个输出文档的字段都包含唯一的按值分组。输出文档还可以包含保存某些<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/group/#std-label-accumulators-group">累加器表达式</a>值的计算字段。</p>
<h3 id="语法"><a href="#语法" class="headerlink" title="语法"></a>语法</h3><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line">  $group:</span><br><span class="line">    &#123;</span><br><span class="line">      _id: &lt;expression&gt;, // Group By Expression</span><br><span class="line">      &lt;field1&gt;: &#123; &lt;accumulator1&gt; : &lt;expression1&gt; &#125;,</span><br><span class="line">      ...</span><br><span class="line">    &#125;</span><br><span class="line"> &#125;</span><br></pre></td></tr></table></figure>



<table>
<thead>
<tr>
<th align="left">Field</th>
<th align="left">Description</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><code>_id</code></td>
<td align="left"><em>必须的</em>。如果指定null或者常量，相当于不分组。</td>
</tr>
<tr>
<td align="left"><code>field</code></td>
<td align="left"><em>可选的.</em> 使用 <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/group/#std-label-accumulators-group">accumulator operators</a>.计算</td>
</tr>
</tbody></table>
<p>常用的 <strong>累加器运算符</strong>：</p>
<table>
<thead>
<tr>
<th align="left">名称</th>
<th align="left">描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/avg/#mongodb-group-grp.-avg"><code>$avg</code></a></td>
<td align="left">返回数值的平均值。忽略非数字值。<em>在5.0版更改</em>：在<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/setWindowFields/#mongodb-pipeline-pipe.-setWindowFields"><code>$setWindowFields</code></a>阶段可用。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/count-accumulator/#mongodb-group-grp.-count"><code>$count</code></a></td>
<td align="left">返回组中的文档数。区别于<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/count/#mongodb-pipeline-pipe.-count"><code>$count</code></a>流水线阶段。<em>5.0版中的新功能</em>：在<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/group/#mongodb-pipeline-pipe.-group"><code>$group</code></a>和 <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/setWindowFields/#mongodb-pipeline-pipe.-setWindowFields"><code>$setWindowFields</code></a>阶段可用。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/first/#mongodb-group-grp.-first"><code>$first</code></a></td>
<td align="left">从每个组的第一个文档返回一个值。仅当文档已排序时才定义顺序。与<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/first-array-element/#mongodb-expression-exp.-first"><code>$first</code></a>数组运算符不同。<em>在5.0版更改</em>：在<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/setWindowFields/#mongodb-pipeline-pipe.-setWindowFields"><code>$setWindowFields</code></a>阶段可用。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/last/#mongodb-group-grp.-last"><code>$last</code></a></td>
<td align="left">从每个组的最后一个文档返回一个值。仅当文档已排序时才定义顺序。与<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/last-array-element/#mongodb-expression-exp.-last"><code>$last</code></a>数组运算符不同。<em>在5.0版更改</em>：在<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/setWindowFields/#mongodb-pipeline-pipe.-setWindowFields"><code>$setWindowFields</code></a>阶段可用。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/max/#mongodb-group-grp.-max"><code>$max</code></a></td>
<td align="left">返回每个组的最高表达式值。<em>在5.0版更改</em>：在<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/setWindowFields/#mongodb-pipeline-pipe.-setWindowFields"><code>$setWindowFields</code></a>阶段可用。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/min/#mongodb-group-grp.-min"><code>$min</code></a></td>
<td align="left">返回每个组的最低表达式值。<em>在5.0版更改</em>：在<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/setWindowFields/#mongodb-pipeline-pipe.-setWindowFields"><code>$setWindowFields</code></a>阶段可用。</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/sum/#mongodb-group-grp.-sum"><code>$sum</code></a></td>
<td align="left">返回数值的总和。忽略非数字值。<em>在5.0版更改</em>：在<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/setWindowFields/#mongodb-pipeline-pipe.-setWindowFields"><code>$setWindowFields</code></a>阶段可用。</td>
</tr>
</tbody></table>
<h3 id="小例子"><a href="#小例子" class="headerlink" title="小例子"></a>小例子</h3><ol>
<li>以下聚合操作按<code>item</code> 字段对文档进行分组，计算每个商品的总销售额并仅返回总销售额大于或等于 100 的商品：</li>
</ol>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">db.sales.aggregate(</span><br><span class="line">  [</span><br><span class="line">    // First Stage</span><br><span class="line">    &#123;</span><br><span class="line">      $group :</span><br><span class="line">        &#123;</span><br><span class="line">          _id : &quot;$item&quot;,</span><br><span class="line">          totalSaleAmount: &#123; $sum: &#123; $multiply: [ &quot;$price&quot;, &quot;$quantity&quot; ] &#125; &#125;</span><br><span class="line">        &#125;</span><br><span class="line">     &#125;,</span><br><span class="line">     // Second Stage</span><br><span class="line">     &#123;</span><br><span class="line">       $match: &#123; &quot;totalSaleAmount&quot;: &#123; $gte: 100 &#125; &#125;</span><br><span class="line">     &#125;</span><br><span class="line">   ]</span><br><span class="line"> )</span><br></pre></td></tr></table></figure>



<ol start="2">
<li>计算计数、总和和平均值</li>
</ol>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line">db.sales.aggregate([</span><br><span class="line">  // First Stage</span><br><span class="line">  &#123;</span><br><span class="line">    $match : &#123; &quot;date&quot;: &#123; $gte: new ISODate(&quot;2014-01-01&quot;), $lt: new ISODate(&quot;2015-01-01&quot;) &#125; &#125;</span><br><span class="line">  &#125;,</span><br><span class="line">  // Second Stage</span><br><span class="line">  &#123;</span><br><span class="line">    $group : &#123;</span><br><span class="line">       _id : &#123; $dateToString: &#123; format: &quot;%Y-%m-%d&quot;, date: &quot;$date&quot; &#125; &#125;,</span><br><span class="line">       totalSaleAmount: &#123; $sum: &#123; $multiply: [ &quot;$price&quot;, &quot;$quantity&quot; ] &#125; &#125;,</span><br><span class="line">       averageQuantity: &#123; $avg: &quot;$quantity&quot; &#125;,</span><br><span class="line">       count: &#123; $sum: 1 &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;,</span><br><span class="line">  // Third Stage</span><br><span class="line">  &#123;</span><br><span class="line">    $sort : &#123; totalSaleAmount: -1 &#125;</span><br><span class="line">  &#125;</span><br><span class="line"> ])</span><br></pre></td></tr></table></figure>

<ul>
<li><p>第一阶段：</p>
<p>该<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/match/#mongodb-pipeline-pipe.-match"><code>$match</code></a>阶段对文档进行过滤，仅将 2014 年的文档传递到下一阶段。</p>
</li>
<li><p>第二阶段：</p>
<p>该<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/group/#mongodb-pipeline-pipe.-group"><code>$group</code></a>阶段按日期对单据进行分组，并计算每组单据的总销售额、平均数量和总数。</p>
</li>
<li><p>第三阶段：</p>
<p>该<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/sort/#mongodb-pipeline-pipe.-sort"><code>$sort</code></a>阶段按每个组的总销售额以降序对结果进行排序。</p>
</li>
</ul>
<p>相当于如下SQL</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">SELECT</span> <span class="type">date</span>,</span><br><span class="line">       <span class="built_in">Sum</span>(( price <span class="operator">*</span> quantity )) <span class="keyword">AS</span> totalSaleAmount,</span><br><span class="line">       <span class="built_in">Avg</span>(quantity)             <span class="keyword">AS</span> averageQuantity,</span><br><span class="line">       <span class="built_in">Count</span>(<span class="operator">*</span>)                  <span class="keyword">AS</span> Count</span><br><span class="line"><span class="keyword">FROM</span>   sales</span><br><span class="line"><span class="keyword">GROUP</span>  <span class="keyword">BY</span> <span class="type">Date</span>(<span class="type">date</span>)</span><br><span class="line"><span class="keyword">ORDER</span>  <span class="keyword">BY</span> totalSaleAmount <span class="keyword">DESC</span></span><br></pre></td></tr></table></figure>







<h2 id="lookup实现关联查询"><a href="#lookup实现关联查询" class="headerlink" title="lookup实现关联查询"></a>lookup实现关联查询</h2><blockquote>
<p>参考<a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/">官方文档</a></p>
</blockquote>
<h3 id="含义与作用-1"><a href="#含义与作用-1" class="headerlink" title="含义与作用"></a>含义与作用</h3><p>对<em>同一</em> 数据库中的未分片集合执行左外部联接，以从“联接”集合中过滤文档以进行处理。对于每个输入文档，lookup阶段会添加一个新的数组字段，其元素是“joined”集合中的匹配文档。该lookup阶段将这些重塑的文档传递到下一阶段。</p>
<h3 id="语法-1"><a href="#语法-1" class="headerlink" title="语法"></a>语法</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line"><span class="meta prompt_">   $</span><span class="language-bash">lookup:</span></span><br><span class="line">     &#123;</span><br><span class="line">       from: &lt;collection to join&gt;,</span><br><span class="line">       localField: &lt;field from the input documents&gt;,</span><br><span class="line">       foreignField: &lt;field from the documents of the &quot;from&quot; collection&gt;,</span><br><span class="line">       as: &lt;output array field&gt;</span><br><span class="line">     &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p>具体的字段的含义是：</p>
<table>
<thead>
<tr>
<th align="left">Field</th>
<th align="left">Description</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#std-label-lookup-eq-from">from</a></td>
<td align="left">Specifies the collection in the <em>same</em> database to perform the join with. The <code>from</code> collection cannot be sharded. For details, see <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#std-label-lookup-sharded-collections">Sharded Collection Restrictions</a>.</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#std-label-lookup-eq-localField">localField</a></td>
<td align="left">Specifies the field from the documents input to the <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#mongodb-pipeline-pipe.-lookup"><code>$lookup</code></a> stage. <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#mongodb-pipeline-pipe.-lookup"><code>$lookup</code></a> performs an equality match on the <code>localField</code> to the <code>foreignField</code> from the documents of the <code>from</code> collection. If an input document does not contain the <code>localField</code>, the <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#mongodb-pipeline-pipe.-lookup"><code>$lookup</code></a> treats the field as having a value of <code>null</code> for matching purposes.</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#std-label-lookup-eq-foreignField">foreignField</a></td>
<td align="left">Specifies the field from the documents in the <code>from</code> collection. <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#mongodb-pipeline-pipe.-lookup"><code>$lookup</code></a> performs an equality match on the <code>foreignField</code> to the <code>localField</code> from the input documents. If a document in the <code>from</code> collection does not contain the <code>foreignField</code>, the <a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#mongodb-pipeline-pipe.-lookup"><code>$lookup</code></a> treats the value as <code>null</code> for matching purposes.</td>
</tr>
<tr>
<td align="left"><a target="_blank" rel="noopener" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#std-label-lookup-eq-as">as</a></td>
<td align="left">Specifies the name of the new array field to add to the input documents. The new array field contains the matching documents from the <code>from</code> collection. If the specified name already exists in the input document, the existing field is <em>overwritten</em>.</td>
</tr>
</tbody></table>
<p>上面的查询语句等效为：</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span>, <span class="operator">&lt;</span>output <span class="keyword">array</span> field<span class="operator">&gt;</span></span><br><span class="line"><span class="keyword">FROM</span> collection</span><br><span class="line"><span class="keyword">WHERE</span> <span class="operator">&lt;</span>output <span class="keyword">array</span> field<span class="operator">&gt;</span> <span class="keyword">IN</span> (</span><br><span class="line">   <span class="keyword">SELECT</span> <span class="operator">*</span></span><br><span class="line">   <span class="keyword">FROM</span> <span class="operator">&lt;</span>collection <span class="keyword">to</span> <span class="keyword">join</span><span class="operator">&gt;</span></span><br><span class="line">   <span class="keyword">WHERE</span> <span class="operator">&lt;</span>foreignField<span class="operator">&gt;</span> <span class="operator">=</span> <span class="operator">&lt;</span>collection.localField<span class="operator">&gt;</span></span><br><span class="line">);</span><br></pre></td></tr></table></figure>



<h3 id="小例子-1"><a href="#小例子-1" class="headerlink" title="小例子"></a>小例子</h3><p>订单表</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">db.orders.insertMany( [</span><br><span class="line">   &#123; &quot;_id&quot; : 1, &quot;item&quot; : &quot;almonds&quot;, &quot;price&quot; : 12, &quot;quantity&quot; : 2 &#125;,</span><br><span class="line">   &#123; &quot;_id&quot; : 2, &quot;item&quot; : &quot;pecans&quot;, &quot;price&quot; : 20, &quot;quantity&quot; : 1 &#125;,</span><br><span class="line">   &#123; &quot;_id&quot; : 3  &#125;</span><br><span class="line">] )</span><br></pre></td></tr></table></figure>



<p>库存表</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">db.inventory.insertMany( [</span><br><span class="line">   &#123; &quot;_id&quot; : 1, &quot;sku&quot; : &quot;almonds&quot;, &quot;description&quot;: &quot;product 1&quot;, &quot;instock&quot; : 120 &#125;,</span><br><span class="line">   &#123; &quot;_id&quot; : 2, &quot;sku&quot; : &quot;bread&quot;, &quot;description&quot;: &quot;product 2&quot;, &quot;instock&quot; : 80 &#125;,</span><br><span class="line">   &#123; &quot;_id&quot; : 3, &quot;sku&quot; : &quot;cashews&quot;, &quot;description&quot;: &quot;product 3&quot;, &quot;instock&quot; : 60 &#125;,</span><br><span class="line">   &#123; &quot;_id&quot; : 4, &quot;sku&quot; : &quot;pecans&quot;, &quot;description&quot;: &quot;product 4&quot;, &quot;instock&quot; : 70 &#125;,</span><br><span class="line">   &#123; &quot;_id&quot; : 5, &quot;sku&quot;: null, &quot;description&quot;: &quot;Incomplete&quot; &#125;,</span><br><span class="line">   &#123; &quot;_id&quot; : 6 &#125;</span><br><span class="line">] )</span><br></pre></td></tr></table></figure>



<p>查询语句</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">db.orders.aggregate( [</span><br><span class="line">   &#123;</span><br><span class="line">     $lookup:</span><br><span class="line">       &#123;</span><br><span class="line">         from: &quot;inventory&quot;,</span><br><span class="line">         localField: &quot;item&quot;,</span><br><span class="line">         foreignField: &quot;sku&quot;,</span><br><span class="line">         as: &quot;inventory_docs&quot;</span><br><span class="line">       &#125;</span><br><span class="line">  &#125;</span><br><span class="line">] )</span><br></pre></td></tr></table></figure>



<p>查询结果</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">   <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;item&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;almonds&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;price&quot;</span> <span class="punctuation">:</span> <span class="number">12</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;quantity&quot;</span> <span class="punctuation">:</span> <span class="number">2</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;inventory_docs&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">      <span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="number">1</span><span class="punctuation">,</span> <span class="attr">&quot;sku&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;almonds&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;description&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;product 1&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;instock&quot;</span> <span class="punctuation">:</span> <span class="number">120</span> <span class="punctuation">&#125;</span></span><br><span class="line">   <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">   <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="number">2</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;item&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;pecans&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;price&quot;</span> <span class="punctuation">:</span> <span class="number">20</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;quantity&quot;</span> <span class="punctuation">:</span> <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;inventory_docs&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">      <span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="number">4</span><span class="punctuation">,</span> <span class="attr">&quot;sku&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;pecans&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;description&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;product 4&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;instock&quot;</span> <span class="punctuation">:</span> <span class="number">70</span> <span class="punctuation">&#125;</span></span><br><span class="line">   <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">   <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="number">3</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;inventory_docs&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">      <span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span> <span class="attr">&quot;sku&quot;</span> <span class="punctuation">:</span> <span class="literal"><span class="keyword">null</span></span><span class="punctuation">,</span> <span class="attr">&quot;description&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;Incomplete&quot;</span> <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">      <span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="number">6</span> <span class="punctuation">&#125;</span></span><br><span class="line">   <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>








      
    </div>

    
    
    

    <footer class="post-footer">
        <div class="post-eof"></div>
      
    </footer>
  </article>
</div>




  <nav class="pagination">
    <a class="extend prev" rel="prev" title="上一页" aria-label="上一页" href="/page/20/"><i class="fa fa-angle-left"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/20/">20</a><span class="page-number current">21</span><a class="page-number" href="/page/22/">22</a><span class="space">&hellip;</span><a class="page-number" href="/page/26/">26</a><a class="extend next" rel="next" title="下一页" aria-label="下一页" href="/page/22/"><i class="fa fa-angle-right"></i></a>
  </nav>

</div>
  </main>

  <footer class="footer">
    <div class="footer-inner">


<div class="copyright">
  &copy; 
  <span itemprop="copyrightYear">2023</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">SongyangJi</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io/" rel="noopener" target="_blank">Hexo</a> & <a href="https://theme-next.js.org/muse/" rel="noopener" target="_blank">NexT.Muse</a> 强力驱动
  </div>

    </div>
  </footer>

  
  <div class="toggle sidebar-toggle" role="button">
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
  </div>
  <div class="sidebar-dimmer"></div>
  <div class="back-to-top" role="button" aria-label="返回顶部">
    <i class="fa fa-arrow-up fa-lg"></i>
    <span>0%</span>
  </div>

<noscript>
  <div class="noscript-warning">Theme NexT works best with JavaScript enabled</div>
</noscript>


  
  <script src="https://cdnjs.cloudflare.com/ajax/libs/animejs/3.2.1/anime.min.js" integrity="sha256-XL2inqUJaslATFnHdJOi9GfQ60on8Wx1C2H8DYiN1xY=" crossorigin="anonymous"></script>
<script src="/js/comments.js"></script><script src="/js/utils.js"></script><script src="/js/motion.js"></script><script src="/js/schemes/muse.js"></script><script src="/js/next-boot.js"></script>

  <script src="https://cdnjs.cloudflare.com/ajax/libs/hexo-generator-searchdb/1.4.1/search.js" integrity="sha256-1kfA5uHPf65M5cphT2dvymhkuyHPQp5A53EGZOnOLmc=" crossorigin="anonymous"></script>
<script src="/js/third-party/search/local-search.js"></script>





  





</body>
</html>
